2024-11-14 19:34:42,270 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-14 19:34:42,287 main DEBUG Took 0.014031 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 19:34:42,288 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 19:34:42,288 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 19:34:42,295 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 19:34:42,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,303 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 19:34:42,315 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,317 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,318 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,319 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,320 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,321 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,322 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,322 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,323 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,323 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,323 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,324 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,324 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,325 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,325 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,325 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,326 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 19:34:42,326 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,326 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 19:34:42,328 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 19:34:42,329 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 19:34:42,331 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 19:34:42,331 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 19:34:42,332 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 19:34:42,333 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 19:34:42,340 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 19:34:42,342 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 19:34:42,344 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 19:34:42,344 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 19:34:42,344 main DEBUG createAppenders(={Console}) 2024-11-14 19:34:42,345 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-14 19:34:42,345 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-14 19:34:42,346 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-14 19:34:42,346 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 19:34:42,346 main DEBUG OutputStream closed 2024-11-14 19:34:42,347 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 19:34:42,347 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 19:34:42,347 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-14 19:34:42,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 19:34:42,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 19:34:42,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 19:34:42,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 19:34:42,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 19:34:42,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 19:34:42,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 19:34:42,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 19:34:42,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 19:34:42,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 19:34:42,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 19:34:42,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 19:34:42,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 19:34:42,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 19:34:42,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 19:34:42,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 19:34:42,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 19:34:42,426 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 19:34:42,428 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 19:34:42,428 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-14 19:34:42,429 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 19:34:42,429 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-14T19:34:42,661 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657 2024-11-14 19:34:42,664 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 19:34:42,664 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T19:34:42,672 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-14T19:34:42,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T19:34:42,720 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389, deleteOnExit=true 2024-11-14T19:34:42,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T19:34:42,722 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/test.cache.data in system properties and HBase conf 2024-11-14T19:34:42,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T19:34:42,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/hadoop.log.dir in system properties and HBase conf 2024-11-14T19:34:42,725 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T19:34:42,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T19:34:42,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T19:34:42,837 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T19:34:42,933 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T19:34:42,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T19:34:42,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T19:34:42,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T19:34:42,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T19:34:42,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T19:34:42,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T19:34:42,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T19:34:42,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T19:34:42,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T19:34:42,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/nfs.dump.dir in system properties and HBase conf 2024-11-14T19:34:42,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/java.io.tmpdir in system properties and HBase conf 2024-11-14T19:34:42,947 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T19:34:42,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T19:34:42,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T19:34:43,918 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T19:34:43,992 INFO [Time-limited test {}] log.Log(170): Logging initialized @2545ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T19:34:44,061 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T19:34:44,121 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T19:34:44,143 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T19:34:44,144 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T19:34:44,145 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T19:34:44,160 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T19:34:44,163 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/hadoop.log.dir/,AVAILABLE} 2024-11-14T19:34:44,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T19:34:44,354 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/java.io.tmpdir/jetty-localhost-45097-hadoop-hdfs-3_4_1-tests_jar-_-any-10782156745099357689/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T19:34:44,361 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:45097} 2024-11-14T19:34:44,362 INFO [Time-limited test {}] server.Server(415): Started @2915ms 2024-11-14T19:34:44,865 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T19:34:44,876 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T19:34:44,878 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T19:34:44,878 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T19:34:44,879 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T19:34:44,880 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/hadoop.log.dir/,AVAILABLE} 2024-11-14T19:34:44,881 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T19:34:45,015 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@42a79998{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/java.io.tmpdir/jetty-localhost-38541-hadoop-hdfs-3_4_1-tests_jar-_-any-5922566180074218476/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T19:34:45,017 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:38541} 2024-11-14T19:34:45,018 INFO [Time-limited test {}] server.Server(415): Started @3571ms 2024-11-14T19:34:45,128 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T19:34:46,567 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389/data/data2/current/BP-1729087364-172.17.0.2-1731612883524/current, will proceed with Du for space computation calculation, 2024-11-14T19:34:46,568 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389/data/data1/current/BP-1729087364-172.17.0.2-1731612883524/current, will proceed with Du for space computation calculation, 2024-11-14T19:34:46,699 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T19:34:46,803 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6579eed040972205 with lease ID 0xab96cab7c1176f0f: Processing first storage report for DS-5d2be3a7-fdbb-40cc-bf8e-55ca298cdcfe from datanode DatanodeRegistration(127.0.0.1:43995, datanodeUuid=61f6364d-ad36-44dd-b7fe-2d5da06a630e, infoPort=40405, infoSecurePort=0, ipcPort=35327, storageInfo=lv=-57;cid=testClusterID;nsid=1789418891;c=1731612883524) 2024-11-14T19:34:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6579eed040972205 with lease ID 0xab96cab7c1176f0f: from storage DS-5d2be3a7-fdbb-40cc-bf8e-55ca298cdcfe node DatanodeRegistration(127.0.0.1:43995, datanodeUuid=61f6364d-ad36-44dd-b7fe-2d5da06a630e, infoPort=40405, infoSecurePort=0, ipcPort=35327, storageInfo=lv=-57;cid=testClusterID;nsid=1789418891;c=1731612883524), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T19:34:46,810 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6579eed040972205 with lease ID 0xab96cab7c1176f0f: Processing first storage report for DS-e9ed76b8-2828-40a4-98f9-967b28c555a2 from datanode DatanodeRegistration(127.0.0.1:43995, datanodeUuid=61f6364d-ad36-44dd-b7fe-2d5da06a630e, infoPort=40405, infoSecurePort=0, ipcPort=35327, storageInfo=lv=-57;cid=testClusterID;nsid=1789418891;c=1731612883524) 2024-11-14T19:34:46,811 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6579eed040972205 with lease ID 0xab96cab7c1176f0f: from storage DS-e9ed76b8-2828-40a4-98f9-967b28c555a2 node DatanodeRegistration(127.0.0.1:43995, datanodeUuid=61f6364d-ad36-44dd-b7fe-2d5da06a630e, infoPort=40405, infoSecurePort=0, ipcPort=35327, storageInfo=lv=-57;cid=testClusterID;nsid=1789418891;c=1731612883524), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T19:34:46,865 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657 2024-11-14T19:34:47,113 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389/zookeeper_0, clientPort=51734, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T19:34:47,132 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51734 2024-11-14T19:34:47,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T19:34:47,169 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T19:34:47,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741825_1001 (size=7) 2024-11-14T19:34:48,012 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 with version=8 2024-11-14T19:34:48,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/hbase-staging 2024-11-14T19:34:48,163 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T19:34:48,522 INFO [Time-limited test {}] client.ConnectionUtils(128): master/867b237d0fa7:0 server-side Connection retries=45 2024-11-14T19:34:48,537 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T19:34:48,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T19:34:48,550 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T19:34:48,551 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T19:34:48,551 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T19:34:48,804 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T19:34:48,902 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T19:34:48,916 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T19:34:48,923 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T19:34:48,958 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 9613 (auto-detected) 2024-11-14T19:34:48,960 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T19:34:48,986 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38019 2024-11-14T19:34:49,021 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38019 connecting to ZooKeeper ensemble=127.0.0.1:51734 2024-11-14T19:34:49,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380190x0, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T19:34:49,179 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38019-0x1013c058da80000 connected 2024-11-14T19:34:49,319 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T19:34:49,323 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T19:34:49,355 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T19:34:49,360 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041, hbase.cluster.distributed=false 2024-11-14T19:34:49,431 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T19:34:49,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38019 2024-11-14T19:34:49,466 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38019 2024-11-14T19:34:49,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38019 2024-11-14T19:34:49,499 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38019 2024-11-14T19:34:49,501 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38019 2024-11-14T19:34:49,690 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/867b237d0fa7:0 server-side Connection retries=45 2024-11-14T19:34:49,692 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T19:34:49,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T19:34:49,693 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T19:34:49,693 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T19:34:49,694 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T19:34:49,698 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T19:34:49,703 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T19:34:49,705 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38601 2024-11-14T19:34:49,709 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38601 connecting to ZooKeeper ensemble=127.0.0.1:51734 2024-11-14T19:34:49,715 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T19:34:49,723 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T19:34:49,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386010x0, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T19:34:49,769 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386010x0, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T19:34:49,769 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38601-0x1013c058da80001 connected 2024-11-14T19:34:49,775 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T19:34:49,793 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T19:34:49,797 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T19:34:49,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T19:34:49,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38601 2024-11-14T19:34:49,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38601 2024-11-14T19:34:49,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38601 2024-11-14T19:34:49,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38601 2024-11-14T19:34:49,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38601 2024-11-14T19:34:49,853 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;867b237d0fa7:38019 2024-11-14T19:34:49,855 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/867b237d0fa7,38019,1731612888274 2024-11-14T19:34:49,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T19:34:49,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T19:34:49,880 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/867b237d0fa7,38019,1731612888274 2024-11-14T19:34:49,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:49,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T19:34:49,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:49,943 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T19:34:49,945 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/867b237d0fa7,38019,1731612888274 from backup master directory 2024-11-14T19:34:49,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T19:34:49,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/867b237d0fa7,38019,1731612888274 2024-11-14T19:34:49,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T19:34:49,954 WARN [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T19:34:49,954 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=867b237d0fa7,38019,1731612888274 2024-11-14T19:34:49,957 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T19:34:49,969 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T19:34:50,101 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/hbase.id] with ID: e950080f-d9c0-4d61-bb4c-96168b3f7750 2024-11-14T19:34:50,101 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/.tmp/hbase.id 2024-11-14T19:34:50,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741826_1002 (size=42) 2024-11-14T19:34:50,160 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/.tmp/hbase.id]:[hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/hbase.id] 2024-11-14T19:34:50,250 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T19:34:50,258 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T19:34:50,288 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-11-14T19:34:50,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:50,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:50,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741827_1003 (size=196) 2024-11-14T19:34:50,397 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:34:50,402 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T19:34:50,438 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:50,448 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T19:34:50,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741828_1004 (size=1189) 2024-11-14T19:34:50,931 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store 2024-11-14T19:34:50,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741829_1005 (size=34) 2024-11-14T19:34:50,976 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T19:34:50,980 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:34:50,982 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T19:34:50,982 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:34:50,982 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:34:50,985 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T19:34:50,985 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:34:50,986 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:34:50,987 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731612890982Disabling compacts and flushes for region at 1731612890982Disabling writes for close at 1731612890985 (+3 ms)Writing region close event to WAL at 1731612890986 (+1 ms)Closed at 1731612890986 2024-11-14T19:34:50,991 WARN [master/867b237d0fa7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/.initializing 2024-11-14T19:34:50,991 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/WALs/867b237d0fa7,38019,1731612888274 2024-11-14T19:34:51,007 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T19:34:51,034 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=867b237d0fa7%2C38019%2C1731612888274, suffix=, logDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/WALs/867b237d0fa7,38019,1731612888274, archiveDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/oldWALs, maxLogs=10 2024-11-14T19:34:51,074 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/WALs/867b237d0fa7,38019,1731612888274/867b237d0fa7%2C38019%2C1731612888274.1731612891041, exclude list is [], retry=0 2024-11-14T19:34:51,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43995,DS-5d2be3a7-fdbb-40cc-bf8e-55ca298cdcfe,DISK] 2024-11-14T19:34:51,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T19:34:51,153 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/WALs/867b237d0fa7,38019,1731612888274/867b237d0fa7%2C38019%2C1731612888274.1731612891041 2024-11-14T19:34:51,154 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40405:40405)] 2024-11-14T19:34:51,155 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:34:51,156 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:34:51,164 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,166 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T19:34:51,276 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:51,280 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:51,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,291 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T19:34:51,292 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:51,295 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:34:51,296 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,310 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T19:34:51,310 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:51,313 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:34:51,313 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T19:34:51,325 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:51,327 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:34:51,328 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,334 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,337 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,353 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,354 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,365 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T19:34:51,375 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T19:34:51,405 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:34:51,408 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71234950, jitterRate=0.0614834725856781}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T19:34:51,419 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731612891184Initializing all the Stores at 1731612891187 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731612891188 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612891189 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612891189Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612891189Cleaning up temporary data from old regions at 1731612891354 (+165 ms)Region opened successfully at 1731612891418 (+64 ms) 2024-11-14T19:34:51,426 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T19:34:51,500 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c015a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=867b237d0fa7/172.17.0.2:0 2024-11-14T19:34:51,554 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T19:34:51,576 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T19:34:51,576 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T19:34:51,582 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T19:34:51,584 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-14T19:34:51,593 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-14T19:34:51,594 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T19:34:51,670 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T19:34:51,682 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T19:34:51,732 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T19:34:51,736 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T19:34:51,739 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T19:34:51,749 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T19:34:51,753 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T19:34:51,759 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T19:34:51,770 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T19:34:51,776 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T19:34:51,787 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T19:34:51,822 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T19:34:51,833 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T19:34:51,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T19:34:51,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:51,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T19:34:51,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:51,858 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=867b237d0fa7,38019,1731612888274, sessionid=0x1013c058da80000, setting cluster-up flag (Was=false) 2024-11-14T19:34:51,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:51,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:51,924 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T19:34:51,928 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=867b237d0fa7,38019,1731612888274 2024-11-14T19:34:51,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:51,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:51,983 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T19:34:51,987 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=867b237d0fa7,38019,1731612888274 2024-11-14T19:34:51,996 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T19:34:52,044 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(746): ClusterId : e950080f-d9c0-4d61-bb4c-96168b3f7750 2024-11-14T19:34:52,047 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T19:34:52,061 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T19:34:52,061 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T19:34:52,073 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T19:34:52,074 DEBUG [RS:0;867b237d0fa7:38601 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422eb04c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=867b237d0fa7/172.17.0.2:0 2024-11-14T19:34:52,105 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;867b237d0fa7:38601 2024-11-14T19:34:52,109 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T19:34:52,110 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T19:34:52,110 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T19:34:52,110 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T19:34:52,114 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(2659): reportForDuty to master=867b237d0fa7,38019,1731612888274 with port=38601, startcode=1731612889637 2024-11-14T19:34:52,130 DEBUG [RS:0;867b237d0fa7:38601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T19:34:52,130 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T19:34:52,139 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T19:34:52,146 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 867b237d0fa7,38019,1731612888274 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T19:34:52,156 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/867b237d0fa7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T19:34:52,157 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/867b237d0fa7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T19:34:52,157 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/867b237d0fa7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T19:34:52,157 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/867b237d0fa7:0, corePoolSize=5, maxPoolSize=5 2024-11-14T19:34:52,158 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/867b237d0fa7:0, corePoolSize=10, maxPoolSize=10 2024-11-14T19:34:52,158 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,158 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/867b237d0fa7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T19:34:52,159 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,169 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T19:34:52,169 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T19:34:52,177 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731612922177 2024-11-14T19:34:52,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58673, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T19:34:52,179 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T19:34:52,179 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:52,180 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T19:34:52,181 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T19:34:52,185 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T19:34:52,186 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T19:34:52,187 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T19:34:52,187 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T19:34:52,192 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38019 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:52,216 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,225 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T19:34:52,227 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T19:34:52,228 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T19:34:52,229 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T19:34:52,229 WARN [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T19:34:52,236 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T19:34:52,237 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T19:34:52,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741831_1007 (size=1321) 2024-11-14T19:34:52,242 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T19:34:52,242 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/867b237d0fa7:0:becomeActiveMaster-HFileCleaner.large.0-1731612892239,5,FailOnTimeoutGroup] 2024-11-14T19:34:52,243 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:34:52,250 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/867b237d0fa7:0:becomeActiveMaster-HFileCleaner.small.0-1731612892243,5,FailOnTimeoutGroup] 2024-11-14T19:34:52,250 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,251 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T19:34:52,253 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,253 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741832_1008 (size=32) 2024-11-14T19:34:52,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:34:52,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T19:34:52,309 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T19:34:52,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:52,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:52,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T19:34:52,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T19:34:52,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:52,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:52,325 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T19:34:52,331 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(2659): reportForDuty to master=867b237d0fa7,38019,1731612888274 with port=38601, startcode=1731612889637 2024-11-14T19:34:52,338 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38019 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 867b237d0fa7,38601,1731612889637 2024-11-14T19:34:52,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T19:34:52,340 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:52,341 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38019 {}] master.ServerManager(517): Registering regionserver=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:52,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:52,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T19:34:52,365 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T19:34:52,366 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:34:52,367 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36419 2024-11-14T19:34:52,367 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T19:34:52,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:52,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:52,369 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T19:34:52,375 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740 2024-11-14T19:34:52,376 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740 2024-11-14T19:34:52,378 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T19:34:52,379 DEBUG [RS:0;867b237d0fa7:38601 {}] zookeeper.ZKUtil(111): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/867b237d0fa7,38601,1731612889637 2024-11-14T19:34:52,379 WARN [RS:0;867b237d0fa7:38601 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T19:34:52,380 INFO [RS:0;867b237d0fa7:38601 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T19:34:52,380 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/WALs/867b237d0fa7,38601,1731612889637 2024-11-14T19:34:52,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T19:34:52,384 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T19:34:52,387 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:34:52,393 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [867b237d0fa7,38601,1731612889637] 2024-11-14T19:34:52,404 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T19:34:52,450 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:34:52,452 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66243399, jitterRate=-0.012896433472633362}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:34:52,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731612892296Initializing all the Stores at 1731612892300 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731612892300Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731612892301 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612892301Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731612892301Cleaning up temporary data from old regions at 1731612892384 (+83 ms)Region opened successfully at 1731612892461 (+77 ms) 2024-11-14T19:34:52,462 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T19:34:52,462 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T19:34:52,462 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T19:34:52,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T19:34:52,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T19:34:52,463 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T19:34:52,466 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T19:34:52,466 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731612892462Disabling compacts and flushes for region at 1731612892462Disabling writes for close at 1731612892463 (+1 ms)Writing region close event to WAL at 1731612892465 (+2 ms)Closed at 1731612892466 (+1 ms) 2024-11-14T19:34:52,471 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T19:34:52,471 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T19:34:52,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T19:34:52,503 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T19:34:52,509 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T19:34:52,510 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T19:34:52,541 INFO [RS:0;867b237d0fa7:38601 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T19:34:52,541 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,552 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T19:34:52,570 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T19:34:52,573 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,575 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,575 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,575 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,575 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,576 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,576 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/867b237d0fa7:0, corePoolSize=2, maxPoolSize=2 2024-11-14T19:34:52,576 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,577 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,577 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,577 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,578 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,578 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/867b237d0fa7:0, corePoolSize=1, maxPoolSize=1 2024-11-14T19:34:52,578 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/867b237d0fa7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T19:34:52,578 DEBUG [RS:0;867b237d0fa7:38601 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0, corePoolSize=3, maxPoolSize=3 2024-11-14T19:34:52,601 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,602 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,602 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,602 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,603 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,603 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38601,1731612889637-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T19:34:52,639 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T19:34:52,646 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38601,1731612889637-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,647 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,648 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.Replication(171): 867b237d0fa7,38601,1731612889637 started 2024-11-14T19:34:52,664 WARN [867b237d0fa7:38019 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T19:34:52,682 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:52,702 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1482): Serving as 867b237d0fa7,38601,1731612889637, RpcServer on 867b237d0fa7/172.17.0.2:38601, sessionid=0x1013c058da80001 2024-11-14T19:34:52,703 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T19:34:52,704 DEBUG [RS:0;867b237d0fa7:38601 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 867b237d0fa7,38601,1731612889637 2024-11-14T19:34:52,704 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '867b237d0fa7,38601,1731612889637' 2024-11-14T19:34:52,704 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T19:34:52,714 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T19:34:52,716 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T19:34:52,716 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T19:34:52,720 DEBUG [RS:0;867b237d0fa7:38601 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 867b237d0fa7,38601,1731612889637 2024-11-14T19:34:52,722 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '867b237d0fa7,38601,1731612889637' 2024-11-14T19:34:52,722 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T19:34:52,723 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T19:34:52,724 DEBUG [RS:0;867b237d0fa7:38601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T19:34:52,724 INFO [RS:0;867b237d0fa7:38601 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T19:34:52,724 INFO [RS:0;867b237d0fa7:38601 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T19:34:52,831 INFO [RS:0;867b237d0fa7:38601 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T19:34:52,841 INFO [RS:0;867b237d0fa7:38601 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=867b237d0fa7%2C38601%2C1731612889637, suffix=, logDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/WALs/867b237d0fa7,38601,1731612889637, archiveDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/oldWALs, maxLogs=32 2024-11-14T19:34:52,864 DEBUG [RS:0;867b237d0fa7:38601 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/WALs/867b237d0fa7,38601,1731612889637/867b237d0fa7%2C38601%2C1731612889637.1731612892845, exclude list is [], retry=0 2024-11-14T19:34:52,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43995,DS-5d2be3a7-fdbb-40cc-bf8e-55ca298cdcfe,DISK] 2024-11-14T19:34:52,888 INFO [RS:0;867b237d0fa7:38601 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/WALs/867b237d0fa7,38601,1731612889637/867b237d0fa7%2C38601%2C1731612889637.1731612892845 2024-11-14T19:34:52,891 DEBUG [RS:0;867b237d0fa7:38601 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40405:40405)] 2024-11-14T19:34:53,167 DEBUG [867b237d0fa7:38019 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T19:34:53,181 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:53,191 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 867b237d0fa7,38601,1731612889637, state=OPENING 2024-11-14T19:34:53,291 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T19:34:53,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:53,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:34:53,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T19:34:53,304 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T19:34:53,307 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T19:34:53,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:34:53,518 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T19:34:53,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43417, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T19:34:53,574 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T19:34:53,575 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T19:34:53,577 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T19:34:53,585 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=867b237d0fa7%2C38601%2C1731612889637.meta, suffix=.meta, logDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/WALs/867b237d0fa7,38601,1731612889637, archiveDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/oldWALs, maxLogs=32 2024-11-14T19:34:53,614 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/WALs/867b237d0fa7,38601,1731612889637/867b237d0fa7%2C38601%2C1731612889637.meta.1731612893588.meta, exclude list is [], retry=0 2024-11-14T19:34:53,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43995,DS-5d2be3a7-fdbb-40cc-bf8e-55ca298cdcfe,DISK] 2024-11-14T19:34:53,644 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/WALs/867b237d0fa7,38601,1731612889637/867b237d0fa7%2C38601%2C1731612889637.meta.1731612893588.meta 2024-11-14T19:34:53,647 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40405:40405)] 2024-11-14T19:34:53,647 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:34:53,650 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T19:34:53,654 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T19:34:53,661 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T19:34:53,679 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T19:34:53,680 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:34:53,681 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T19:34:53,681 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T19:34:53,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T19:34:53,713 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T19:34:53,713 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:53,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:53,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T19:34:53,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T19:34:53,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:53,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:53,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T19:34:53,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T19:34:53,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:53,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:53,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T19:34:53,734 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T19:34:53,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:53,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T19:34:53,740 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T19:34:53,743 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740 2024-11-14T19:34:53,749 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740 2024-11-14T19:34:53,755 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T19:34:53,756 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T19:34:53,758 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:34:53,764 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T19:34:53,769 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72330263, jitterRate=0.07780490815639496}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:34:53,769 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T19:34:53,772 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731612893682Writing region info on filesystem at 1731612893683 (+1 ms)Initializing all the Stores at 1731612893689 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731612893689Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731612893710 (+21 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612893711 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731612893711Cleaning up temporary data from old regions at 1731612893757 (+46 ms)Running coprocessor post-open hooks at 1731612893770 (+13 ms)Region opened successfully at 1731612893772 (+2 ms) 2024-11-14T19:34:53,782 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731612893492 2024-11-14T19:34:53,798 DEBUG [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T19:34:53,799 INFO [RS_OPEN_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T19:34:53,803 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:53,807 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 867b237d0fa7,38601,1731612889637, state=OPEN 2024-11-14T19:34:53,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T19:34:53,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T19:34:53,855 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T19:34:53,856 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:53,856 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T19:34:53,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T19:34:53,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=867b237d0fa7,38601,1731612889637 in 547 msec 2024-11-14T19:34:53,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T19:34:53,882 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.3820 sec 2024-11-14T19:34:53,883 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T19:34:53,883 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T19:34:53,907 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:53,909 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:53,935 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:53,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39817, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:53,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.9270 sec 2024-11-14T19:34:53,970 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731612893970, completionTime=-1 2024-11-14T19:34:53,975 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T19:34:53,975 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T19:34:54,040 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-14T19:34:54,041 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731612954040 2024-11-14T19:34:54,041 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731613014041 2024-11-14T19:34:54,041 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 65 msec 2024-11-14T19:34:54,043 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38019,1731612888274-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:54,044 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38019,1731612888274-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:54,044 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38019,1731612888274-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:54,046 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-867b237d0fa7:38019, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:54,047 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:54,053 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:54,055 DEBUG [master/867b237d0fa7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T19:34:54,094 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.139sec 2024-11-14T19:34:54,095 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T19:34:54,097 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T19:34:54,098 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T19:34:54,098 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T19:34:54,099 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T19:34:54,099 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38019,1731612888274-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T19:34:54,100 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38019,1731612888274-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T19:34:54,109 DEBUG [master/867b237d0fa7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T19:34:54,110 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T19:34:54,110 INFO [master/867b237d0fa7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=867b237d0fa7,38019,1731612888274-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T19:34:54,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519673c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:54,157 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T19:34:54,157 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T19:34:54,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:54,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:54,175 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672abb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:54,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:54,204 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35126, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:54,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d93e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:54,209 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:54,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:54,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=867b237d0fa7,38019,1731612888274 2024-11-14T19:34:54,269 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=6524 2024-11-14T19:34:54,280 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T19:34:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 867b237d0fa7,38019,1731612888274 2024-11-14T19:34:54,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@485738bd 2024-11-14T19:34:54,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T19:34:54,295 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T19:34:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:34:54,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:34:54,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T19:34:54,322 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T19:34:54,325 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:54,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-14T19:34:54,328 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T19:34:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T19:34:54,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741835_1011 (size=1061) 2024-11-14T19:34:54,363 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:34:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741836_1012 (size=53) 2024-11-14T19:34:54,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:34:54,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 21b827b5bf7b423416191650584b91ae, disabling compactions & flushes 2024-11-14T19:34:54,403 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. after waiting 0 ms 2024-11-14T19:34:54,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,403 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21b827b5bf7b423416191650584b91ae: Waiting for close lock at 1731612894403Disabling compacts and flushes for region at 1731612894403Disabling writes for close at 1731612894403Writing region close event to WAL at 1731612894403Closed at 1731612894403 2024-11-14T19:34:54,406 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T19:34:54,412 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731612894406"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731612894406"}]},"ts":"1731612894406"} 2024-11-14T19:34:54,421 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T19:34:54,424 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T19:34:54,428 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612894425"}]},"ts":"1731612894425"} 2024-11-14T19:34:54,435 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T19:34:54,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, ASSIGN}] 2024-11-14T19:34:54,441 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, ASSIGN 2024-11-14T19:34:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T19:34:54,445 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, ASSIGN; state=OFFLINE, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=false 2024-11-14T19:34:54,598 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21b827b5bf7b423416191650584b91ae, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:54,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, ASSIGN because future has completed 2024-11-14T19:34:54,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:34:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T19:34:54,787 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,788 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:34:54,788 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,789 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:34:54,789 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,789 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,796 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,801 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:34:54,802 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b827b5bf7b423416191650584b91ae columnFamilyName A 2024-11-14T19:34:54,802 DEBUG [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:54,810 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.HStore(327): Store=21b827b5bf7b423416191650584b91ae/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:34:54,810 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,813 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:34:54,814 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b827b5bf7b423416191650584b91ae columnFamilyName B 2024-11-14T19:34:54,814 DEBUG [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:54,815 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.HStore(327): Store=21b827b5bf7b423416191650584b91ae/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:34:54,816 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,828 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:34:54,829 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21b827b5bf7b423416191650584b91ae columnFamilyName C 2024-11-14T19:34:54,829 DEBUG [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:34:54,830 INFO [StoreOpener-21b827b5bf7b423416191650584b91ae-1 {}] regionserver.HStore(327): Store=21b827b5bf7b423416191650584b91ae/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:34:54,831 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,831 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,834 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,835 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,839 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,839 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,841 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:34:54,845 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,872 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:34:54,875 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 21b827b5bf7b423416191650584b91ae; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61976994, jitterRate=-0.07647082209587097}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:34:54,875 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:54,877 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 21b827b5bf7b423416191650584b91ae: Running coprocessor pre-open hook at 1731612894790Writing region info on filesystem at 1731612894790Initializing all the Stores at 1731612894793 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612894793Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612894794 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612894794Cleaning up temporary data from old regions at 1731612894839 (+45 ms)Running coprocessor post-open hooks at 1731612894875 (+36 ms)Region opened successfully at 1731612894876 (+1 ms) 2024-11-14T19:34:54,884 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., pid=6, masterSystemTime=1731612894771 2024-11-14T19:34:54,890 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,890 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:54,894 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21b827b5bf7b423416191650584b91ae, regionState=OPEN, openSeqNum=2, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:54,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:34:54,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T19:34:54,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, ASSIGN in 478 msec 2024-11-14T19:34:54,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T19:34:54,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 in 298 msec 2024-11-14T19:34:54,931 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T19:34:54,932 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612894931"}]},"ts":"1731612894931"} 2024-11-14T19:34:54,939 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T19:34:54,942 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T19:34:54,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 632 msec 2024-11-14T19:34:54,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T19:34:54,966 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:34:54,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ede62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:54,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:54,979 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:54,993 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:54,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ecdd502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:54,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:54,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,003 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175b244f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,032 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,054 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,058 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T19:34:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 867b237d0fa7,38019,1731612888274 2024-11-14T19:34:55,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1faa08f6 2024-11-14T19:34:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T19:34:55,078 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T19:34:55,086 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812e936, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,086 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,087 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,089 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9dd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,095 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,097 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,102 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e500c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,102 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,108 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,119 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,126 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68c12611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,126 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,130 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,154 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a084b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77155d64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,173 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0b2729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,188 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c42c597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bdceae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,194 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b41a1cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,197 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72baa874, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58df653d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,205 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,206 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bee872e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,207 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17944247, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,213 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85187e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,215 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276c4225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46f88841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc6fa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,224 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d0e529e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,227 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aaa86d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a990c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:34:55,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:34:55,243 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:34:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:34:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:34:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d25ecbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:34:55,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:34:55,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:34:55,247 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:34:55,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a74f818, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:34:55,264 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,265 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,281 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,281 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,287 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,291 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,292 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,295 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,295 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:34:55,303 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-14T19:34:55,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T19:34:55,326 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:34:55,336 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:34:55,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:34:55,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:34:55,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:34:55,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:34:55,354 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:34:55,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:34:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T19:34:55,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:34:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:55,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:34:55,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:55,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:34:55,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:55,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:34:55,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:55,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:55,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:55,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:55,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612955600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612955601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T19:34:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612955606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612955625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612955631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/ccceba6b5e6e4505829f3ebcfd7c535a is 50, key is test_row_0/A:col10/1731612895414/Put/seqid=0 2024-11-14T19:34:55,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741837_1013 (size=14341) 2024-11-14T19:34:55,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/ccceba6b5e6e4505829f3ebcfd7c535a 2024-11-14T19:34:55,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:55,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612955746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612955746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612955775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612955777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612955780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:55,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:55,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:55,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/a7ccc427375545a5b42537db21526ca4 is 50, key is test_row_0/B:col10/1731612895414/Put/seqid=0 2024-11-14T19:34:55,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T19:34:55,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:55,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612955984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:55,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741838_1014 (size=12001) 2024-11-14T19:34:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:55,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:55,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612955995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612955996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612955996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612955997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612956304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612956319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612956319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612956325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612956326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/a7ccc427375545a5b42537db21526ca4 2024-11-14T19:34:56,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T19:34:56,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/f1dafd5e957648acaa21121d21289d9e is 50, key is test_row_0/C:col10/1731612895414/Put/seqid=0 2024-11-14T19:34:56,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741839_1015 (size=12001) 2024-11-14T19:34:56,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/f1dafd5e957648acaa21121d21289d9e 2024-11-14T19:34:56,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/ccceba6b5e6e4505829f3ebcfd7c535a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/ccceba6b5e6e4505829f3ebcfd7c535a 2024-11-14T19:34:56,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/ccceba6b5e6e4505829f3ebcfd7c535a, entries=200, sequenceid=13, filesize=14.0 K 2024-11-14T19:34:56,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/a7ccc427375545a5b42537db21526ca4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7ccc427375545a5b42537db21526ca4 2024-11-14T19:34:56,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7ccc427375545a5b42537db21526ca4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T19:34:56,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/f1dafd5e957648acaa21121d21289d9e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/f1dafd5e957648acaa21121d21289d9e 2024-11-14T19:34:56,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/f1dafd5e957648acaa21121d21289d9e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-14T19:34:56,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:56,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 21b827b5bf7b423416191650584b91ae in 1199ms, sequenceid=13, compaction requested=false 2024-11-14T19:34:56,646 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T19:34:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:34:56,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T19:34:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:56,795 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:34:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:34:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:34:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:34:56,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:56,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:56,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/309bcb6fa2c74ba29421f854821f39d7 is 50, key is test_row_0/A:col10/1731612895621/Put/seqid=0 2024-11-14T19:34:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741840_1016 (size=12001) 2024-11-14T19:34:56,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612956904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:56,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:56,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612956918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:56,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612956930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612956954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612956966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612957085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612957104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612957115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612957118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612957118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,304 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/309bcb6fa2c74ba29421f854821f39d7 2024-11-14T19:34:57,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612957304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612957345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612957346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/104d0baed3da43529511e535795c0b72 is 50, key is test_row_0/B:col10/1731612895621/Put/seqid=0 2024-11-14T19:34:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612957376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612957376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,394 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741841_1017 (size=12001) 2024-11-14T19:34:57,418 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/104d0baed3da43529511e535795c0b72 2024-11-14T19:34:57,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/770faf9bf313448f960dbd4cee815824 is 50, key is test_row_0/C:col10/1731612895621/Put/seqid=0 2024-11-14T19:34:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T19:34:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741842_1018 (size=12001) 2024-11-14T19:34:57,511 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/770faf9bf313448f960dbd4cee815824 2024-11-14T19:34:57,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/309bcb6fa2c74ba29421f854821f39d7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/309bcb6fa2c74ba29421f854821f39d7 2024-11-14T19:34:57,558 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/309bcb6fa2c74ba29421f854821f39d7, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T19:34:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/104d0baed3da43529511e535795c0b72 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/104d0baed3da43529511e535795c0b72 2024-11-14T19:34:57,599 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/104d0baed3da43529511e535795c0b72, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T19:34:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/770faf9bf313448f960dbd4cee815824 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/770faf9bf313448f960dbd4cee815824 2024-11-14T19:34:57,622 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/770faf9bf313448f960dbd4cee815824, entries=150, sequenceid=33, filesize=11.7 K 2024-11-14T19:34:57,630 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 21b827b5bf7b423416191650584b91ae in 834ms, sequenceid=33, compaction requested=false 2024-11-14T19:34:57,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:34:57,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:57,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T19:34:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T19:34:57,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T19:34:57,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2830 sec 2024-11-14T19:34:57,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3450 sec 2024-11-14T19:34:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:57,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:34:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:34:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:34:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:57,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:34:57,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:34:57,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/50d50cf61fae4bc089d8380262c88b72 is 50, key is test_row_0/A:col10/1731612896954/Put/seqid=0 2024-11-14T19:34:57,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741843_1019 (size=14341) 2024-11-14T19:34:57,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/50d50cf61fae4bc089d8380262c88b72 2024-11-14T19:34:57,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/74aebc3d657a46978290085cd7a4967d is 50, key is test_row_0/B:col10/1731612896954/Put/seqid=0 2024-11-14T19:34:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612957757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612957764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612957769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612957770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612957781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741844_1020 (size=12001) 2024-11-14T19:34:57,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/74aebc3d657a46978290085cd7a4967d 2024-11-14T19:34:57,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612957884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612957905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e6d24db87a58482baa5c11230ee1b5dd is 50, key is test_row_0/C:col10/1731612896954/Put/seqid=0 2024-11-14T19:34:57,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612957906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612957906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612957907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:57,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:57,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741845_1021 (size=12001) 2024-11-14T19:34:57,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e6d24db87a58482baa5c11230ee1b5dd 2024-11-14T19:34:58,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/50d50cf61fae4bc089d8380262c88b72 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/50d50cf61fae4bc089d8380262c88b72 2024-11-14T19:34:58,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/50d50cf61fae4bc089d8380262c88b72, entries=200, sequenceid=46, filesize=14.0 K 2024-11-14T19:34:58,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/74aebc3d657a46978290085cd7a4967d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/74aebc3d657a46978290085cd7a4967d 2024-11-14T19:34:58,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/74aebc3d657a46978290085cd7a4967d, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T19:34:58,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e6d24db87a58482baa5c11230ee1b5dd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e6d24db87a58482baa5c11230ee1b5dd 2024-11-14T19:34:58,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e6d24db87a58482baa5c11230ee1b5dd, entries=150, sequenceid=46, filesize=11.7 K 2024-11-14T19:34:58,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 21b827b5bf7b423416191650584b91ae in 416ms, sequenceid=46, compaction requested=true 2024-11-14T19:34:58,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:34:58,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:34:58,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:34:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:34:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:34:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:34:58,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:34:58,085 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:34:58,085 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:34:58,090 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:34:58,092 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:34:58,093 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:58,094 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7ccc427375545a5b42537db21526ca4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/104d0baed3da43529511e535795c0b72, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/74aebc3d657a46978290085cd7a4967d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=35.2 K 2024-11-14T19:34:58,096 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a7ccc427375545a5b42537db21526ca4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731612895414 2024-11-14T19:34:58,105 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 104d0baed3da43529511e535795c0b72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731612895597 2024-11-14T19:34:58,106 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 74aebc3d657a46978290085cd7a4967d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731612896919 2024-11-14T19:34:58,111 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:34:58,111 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:34:58,112 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:58,112 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/ccceba6b5e6e4505829f3ebcfd7c535a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/309bcb6fa2c74ba29421f854821f39d7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/50d50cf61fae4bc089d8380262c88b72] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=39.7 K 2024-11-14T19:34:58,114 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccceba6b5e6e4505829f3ebcfd7c535a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731612895414 2024-11-14T19:34:58,115 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 309bcb6fa2c74ba29421f854821f39d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731612895597 2024-11-14T19:34:58,117 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50d50cf61fae4bc089d8380262c88b72, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731612896891 2024-11-14T19:34:58,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:58,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T19:34:58,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:34:58,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612897743, max timestamp=1731612898120Num uniques 859; 2024-11-14T19:34:58,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612897743, max timestamp=1731612898120Num uniques 859; 2024-11-14T19:34:58,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:34:58,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:34:58,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:34:58,161 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612897743, max timestamp=1731612898120Num uniques 859; 2024-11-14T19:34:58,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:34:58,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:34:58,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:34:58,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:34:58,184 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:34:58,200 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#9 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:34:58,201 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#10 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:34:58,202 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/9bff8ad8a05c410ab6a25c5bc7baa920 is 50, key is test_row_0/B:col10/1731612896954/Put/seqid=0 2024-11-14T19:34:58,204 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/1c41ef8790d64552bbcae76f96e2cb68 is 50, key is test_row_0/A:col10/1731612896954/Put/seqid=0 2024-11-14T19:34:58,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/b4f1e62948774839a3d0b99ff1b88f7a is 50, key is test_row_0/A:col10/1731612898120/Put/seqid=0 2024-11-14T19:34:58,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741846_1022 (size=12104) 2024-11-14T19:34:58,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741847_1023 (size=12104) 2024-11-14T19:34:58,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741848_1024 (size=12001) 2024-11-14T19:34:58,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612898247, max timestamp=1731612898354Num uniques 859; 2024-11-14T19:34:58,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:34:58,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612898247, max timestamp=1731612898363Num uniques 859; 2024-11-14T19:34:58,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:34:58,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612898247, max timestamp=1731612898354Num uniques 859; 2024-11-14T19:34:58,395 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:34:58,430 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T19:34:58,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612898448, max timestamp=1731612898521Num uniques 1018; 2024-11-14T19:34:58,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:34:58,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731612898448, max timestamp=1731612898521Num uniques 1018; 2024-11-14T19:34:58,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-14T19:34:58,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612898448, max timestamp=1731612898531Num uniques 1018; 2024-11-14T19:34:58,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:34:58,689 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/9bff8ad8a05c410ab6a25c5bc7baa920 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9bff8ad8a05c410ab6a25c5bc7baa920 2024-11-14T19:34:58,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/b4f1e62948774839a3d0b99ff1b88f7a 2024-11-14T19:34:58,697 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/1c41ef8790d64552bbcae76f96e2cb68 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1c41ef8790d64552bbcae76f96e2cb68 2024-11-14T19:34:58,734 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into 1c41ef8790d64552bbcae76f96e2cb68(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:34:58,734 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:34:58,743 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=13, startTime=1731612898073; duration=0sec 2024-11-14T19:34:58,745 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:34:58,746 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:34:58,746 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:34:58,750 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 9bff8ad8a05c410ab6a25c5bc7baa920(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:34:58,751 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:34:58,751 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=13, startTime=1731612898085; duration=0sec 2024-11-14T19:34:58,751 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:34:58,751 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:34:58,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/c1578d7d361c4d769ecd7ddf129b6c1a is 50, key is test_row_0/B:col10/1731612898120/Put/seqid=0 2024-11-14T19:34:58,756 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:34:58,756 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:34:58,759 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:58,761 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/f1dafd5e957648acaa21121d21289d9e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/770faf9bf313448f960dbd4cee815824, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e6d24db87a58482baa5c11230ee1b5dd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=35.2 K 2024-11-14T19:34:58,762 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1dafd5e957648acaa21121d21289d9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731612895414 2024-11-14T19:34:58,764 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 770faf9bf313448f960dbd4cee815824, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731612895597 2024-11-14T19:34:58,773 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6d24db87a58482baa5c11230ee1b5dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731612896919 2024-11-14T19:34:58,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731612898595, max timestamp=1731612898760Num uniques 1027; 2024-11-14T19:34:58,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T19:34:58,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731612898595, max timestamp=1731612898760Num uniques 1027; 2024-11-14T19:34:58,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-14T19:34:58,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731612898595, max timestamp=1731612898760Num uniques 1009; 2024-11-14T19:34:58,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-14T19:34:58,837 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#13 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:34:58,839 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/3cd427ce120a4aebacdd836508277aec is 50, key is test_row_0/C:col10/1731612896954/Put/seqid=0 2024-11-14T19:34:58,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741849_1025 (size=12001) 2024-11-14T19:34:58,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612898831, max timestamp=1731612898884Num uniques 1086; 2024-11-14T19:34:58,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612898831, max timestamp=1731612898884Num uniques 1086; 2024-11-14T19:34:58,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T19:34:58,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T19:34:58,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612898831, max timestamp=1731612898884Num uniques 1086; 2024-11-14T19:34:58,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T19:34:58,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741850_1026 (size=12104) 2024-11-14T19:34:58,956 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/3cd427ce120a4aebacdd836508277aec as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3cd427ce120a4aebacdd836508277aec 2024-11-14T19:34:58,974 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into 3cd427ce120a4aebacdd836508277aec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:34:58,974 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:34:58,974 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=13, startTime=1731612898085; duration=0sec 2024-11-14T19:34:58,975 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:34:58,975 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:34:59,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731612898976, max timestamp=1731612898997Num uniques 1045; 2024-11-14T19:34:59,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8923, totalHeapSize=20520, min timestamp=1731612898976, max timestamp=1731612899005Num uniques 1045; 2024-11-14T19:34:59,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731612898976, max timestamp=1731612898997Num uniques 1045; 2024-11-14T19:34:59,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T19:34:59,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T19:34:59,011 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47853, new segment data size=8923, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T19:34:59,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731612899045, max timestamp=1731612899101Num uniques 1054; 2024-11-14T19:34:59,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1731612899045, max timestamp=1731612899101Num uniques 1054; 2024-11-14T19:34:59,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-14T19:34:59,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-11-14T19:34:59,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731612899076, max timestamp=1731612899101Num uniques 1054; 2024-11-14T19:34:59,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-11-14T19:34:59,194 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T19:34:59,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/c1578d7d361c4d769ecd7ddf129b6c1a 2024-11-14T19:34:59,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731612899242, max timestamp=1731612899308Num uniques 1113; 2024-11-14T19:34:59,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-11-14T19:34:59,326 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T19:34:59,326 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T19:34:59,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731612899242, max timestamp=1731612899308Num uniques 1113; 2024-11-14T19:34:59,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=14328, suffix heap size=143720,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=313 2024-11-14T19:34:59,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/464d85af0e7e4247abd03f0810822764 is 50, key is test_row_0/C:col10/1731612898120/Put/seqid=0 2024-11-14T19:34:59,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731612899242, max timestamp=1731612899308Num uniques 1063; 2024-11-14T19:34:59,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=14328, suffix heap size=138520,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=313 2024-11-14T19:34:59,334 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T19:34:59,334 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T19:34:59,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T19:34:59,335 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T19:34:59,336 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T19:34:59,336 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T19:34:59,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741851_1027 (size=12001) 2024-11-14T19:34:59,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/464d85af0e7e4247abd03f0810822764 2024-11-14T19:34:59,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/b4f1e62948774839a3d0b99ff1b88f7a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b4f1e62948774839a3d0b99ff1b88f7a 2024-11-14T19:34:59,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b4f1e62948774839a3d0b99ff1b88f7a, entries=150, sequenceid=67, filesize=11.7 K 2024-11-14T19:34:59,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731612899388, max timestamp=1731612899420Num uniques 1172; 2024-11-14T19:34:59,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-14T19:34:59,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731612899388, max timestamp=1731612899420Num uniques 1172; 2024-11-14T19:34:59,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731612899388, max timestamp=1731612899420Num uniques 1172; 2024-11-14T19:34:59,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-14T19:34:59,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-14T19:34:59,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/c1578d7d361c4d769ecd7ddf129b6c1a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c1578d7d361c4d769ecd7ddf129b6c1a 2024-11-14T19:34:59,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c1578d7d361c4d769ecd7ddf129b6c1a, entries=150, sequenceid=67, filesize=11.7 K 2024-11-14T19:34:59,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/464d85af0e7e4247abd03f0810822764 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/464d85af0e7e4247abd03f0810822764 2024-11-14T19:34:59,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/464d85af0e7e4247abd03f0810822764, entries=150, sequenceid=67, filesize=11.7 K 2024-11-14T19:34:59,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=87.22 KB/89310 for 21b827b5bf7b423416191650584b91ae in 1326ms, sequenceid=67, compaction requested=false 2024-11-14T19:34:59,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:34:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T19:34:59,506 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:34:59,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:34:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-14T19:34:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:34:59,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=100.63 KB heapSize=279.47 KB 2024-11-14T19:34:59,514 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:34:59,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:34:59,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:34:59,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:34:59,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:34:59,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:34:59,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:34:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T19:34:59,517 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:34:59,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:34:59,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/89ab5e127be24b5ebdee1beaccda3c4d is 50, key is test_row_0/A:col10/1731612899502/Put/seqid=0 2024-11-14T19:34:59,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741852_1028 (size=12151) 2024-11-14T19:34:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T19:34:59,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612959580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612959610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612959611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612959629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612959606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:34:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:59,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612959744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612959753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612959754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612959754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612959763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T19:34:59,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:34:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:59,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612959965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612959973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612959973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:34:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612959974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:34:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:34:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:34:59,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:34:59,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:59,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:34:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:34:59,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:34:59,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/89ab5e127be24b5ebdee1beaccda3c4d 2024-11-14T19:35:00,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612959993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/46f41dcbfb254682823aa38744dde1be is 50, key is test_row_0/B:col10/1731612899502/Put/seqid=0 2024-11-14T19:35:00,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741853_1029 (size=12151) 2024-11-14T19:35:00,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/46f41dcbfb254682823aa38744dde1be 2024-11-14T19:35:00,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/1c23ac1e24b54481a35ad03c45ce54d0 is 50, key is test_row_0/C:col10/1731612899502/Put/seqid=0 2024-11-14T19:35:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T19:35:00,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:00,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:00,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741854_1030 (size=12151) 2024-11-14T19:35:00,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/1c23ac1e24b54481a35ad03c45ce54d0 2024-11-14T19:35:00,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/89ab5e127be24b5ebdee1beaccda3c4d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/89ab5e127be24b5ebdee1beaccda3c4d 2024-11-14T19:35:00,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/89ab5e127be24b5ebdee1beaccda3c4d, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T19:35:00,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/46f41dcbfb254682823aa38744dde1be as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/46f41dcbfb254682823aa38744dde1be 2024-11-14T19:35:00,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/46f41dcbfb254682823aa38744dde1be, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T19:35:00,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/1c23ac1e24b54481a35ad03c45ce54d0 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1c23ac1e24b54481a35ad03c45ce54d0 2024-11-14T19:35:00,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612960284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1c23ac1e24b54481a35ad03c45ce54d0, entries=150, sequenceid=221, filesize=11.9 K 2024-11-14T19:35:00,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~279.42 KB/286128, currentSize=80.51 KB/82440 for 21b827b5bf7b423416191650584b91ae in 779ms, sequenceid=221, compaction requested=true 2024-11-14T19:35:00,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:00,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:00,291 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:00,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:00,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:00,292 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:00,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:00,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:00,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,298 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:00,298 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:00,298 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:00,298 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:00,298 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,298 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,298 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9bff8ad8a05c410ab6a25c5bc7baa920, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c1578d7d361c4d769ecd7ddf129b6c1a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/46f41dcbfb254682823aa38744dde1be] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=35.4 K 2024-11-14T19:35:00,298 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1c41ef8790d64552bbcae76f96e2cb68, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b4f1e62948774839a3d0b99ff1b88f7a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/89ab5e127be24b5ebdee1beaccda3c4d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=35.4 K 2024-11-14T19:35:00,300 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bff8ad8a05c410ab6a25c5bc7baa920, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731612896919 2024-11-14T19:35:00,301 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c41ef8790d64552bbcae76f96e2cb68, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731612896919 2024-11-14T19:35:00,303 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c1578d7d361c4d769ecd7ddf129b6c1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731612897743 2024-11-14T19:35:00,303 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4f1e62948774839a3d0b99ff1b88f7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731612897743 2024-11-14T19:35:00,306 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f41dcbfb254682823aa38744dde1be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731612899459 2024-11-14T19:35:00,307 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89ab5e127be24b5ebdee1beaccda3c4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731612899459 2024-11-14T19:35:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:00,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T19:35:00,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:00,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:00,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:00,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:00,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:00,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:00,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:00,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,358 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:00,360 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/774d2424330646f1bc0edb04e7895be9 is 50, key is test_row_0/A:col10/1731612899502/Put/seqid=0 2024-11-14T19:35:00,371 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:00,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/2308d1f5a42b404ab28ca9edd5065dc2 is 50, key is test_row_0/A:col10/1731612900330/Put/seqid=0 2024-11-14T19:35:00,374 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/8d83012693c049f68dc16af714f427c5 is 50, key is test_row_0/B:col10/1731612899502/Put/seqid=0 2024-11-14T19:35:00,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741855_1031 (size=12357) 2024-11-14T19:35:00,414 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/774d2424330646f1bc0edb04e7895be9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/774d2424330646f1bc0edb04e7895be9 2024-11-14T19:35:00,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741856_1032 (size=12357) 2024-11-14T19:35:00,450 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/8d83012693c049f68dc16af714f427c5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/8d83012693c049f68dc16af714f427c5 2024-11-14T19:35:00,458 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into 774d2424330646f1bc0edb04e7895be9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:00,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741857_1033 (size=19321) 2024-11-14T19:35:00,459 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:00,459 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=13, startTime=1731612900291; duration=0sec 2024-11-14T19:35:00,460 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:00,460 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:00,460 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:00,463 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:00,463 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:00,463 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,464 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3cd427ce120a4aebacdd836508277aec, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/464d85af0e7e4247abd03f0810822764, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1c23ac1e24b54481a35ad03c45ce54d0] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=35.4 K 2024-11-14T19:35:00,465 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cd427ce120a4aebacdd836508277aec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731612896919 2024-11-14T19:35:00,466 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464d85af0e7e4247abd03f0810822764, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1731612897743 2024-11-14T19:35:00,466 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c23ac1e24b54481a35ad03c45ce54d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731612899459 2024-11-14T19:35:00,486 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 8d83012693c049f68dc16af714f427c5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:00,486 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:00,486 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=13, startTime=1731612900292; duration=0sec 2024-11-14T19:35:00,487 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:00,487 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:00,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:00,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,495 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,508 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:00,509 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e2a7fdb41cd44ce8936b06f3ec449a20 is 50, key is test_row_0/C:col10/1731612899502/Put/seqid=0 2024-11-14T19:35:00,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612960525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612960530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612960532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612960534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741858_1034 (size=12357) 2024-11-14T19:35:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,572 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e2a7fdb41cd44ce8936b06f3ec449a20 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e2a7fdb41cd44ce8936b06f3ec449a20 2024-11-14T19:35:00,590 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into e2a7fdb41cd44ce8936b06f3ec449a20(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:00,590 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:00,590 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=13, startTime=1731612900292; duration=0sec 2024-11-14T19:35:00,590 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:00,590 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:00,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612960643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:00,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:00,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612960654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612960654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612960654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T19:35:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612960804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:00,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612960853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/2308d1f5a42b404ab28ca9edd5065dc2 2024-11-14T19:35:00,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612960874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612960875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:00,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612960874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:00,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/016721c5794041c18824db5e36a1dc30 is 50, key is test_row_0/B:col10/1731612900330/Put/seqid=0 2024-11-14T19:35:00,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:00,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:00,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:00,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741859_1035 (size=12151) 2024-11-14T19:35:00,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/016721c5794041c18824db5e36a1dc30 2024-11-14T19:35:01,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/1cad4c79100443c7a644445cc8e00be3 is 50, key is test_row_0/C:col10/1731612900330/Put/seqid=0 2024-11-14T19:35:01,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741860_1036 (size=12151) 2024-11-14T19:35:01,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:01,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:01,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612961164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612961194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612961194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612961194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:01,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:01,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:01,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:01,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/1cad4c79100443c7a644445cc8e00be3 2024-11-14T19:35:01,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/2308d1f5a42b404ab28ca9edd5065dc2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2308d1f5a42b404ab28ca9edd5065dc2 2024-11-14T19:35:01,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2308d1f5a42b404ab28ca9edd5065dc2, entries=300, sequenceid=239, filesize=18.9 K 2024-11-14T19:35:01,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/016721c5794041c18824db5e36a1dc30 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/016721c5794041c18824db5e36a1dc30 2024-11-14T19:35:01,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/016721c5794041c18824db5e36a1dc30, entries=150, sequenceid=239, filesize=11.9 K 2024-11-14T19:35:01,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/1cad4c79100443c7a644445cc8e00be3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1cad4c79100443c7a644445cc8e00be3 2024-11-14T19:35:01,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1cad4c79100443c7a644445cc8e00be3, entries=150, sequenceid=239, filesize=11.9 K 2024-11-14T19:35:01,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 21b827b5bf7b423416191650584b91ae in 1270ms, sequenceid=239, compaction requested=false 2024-11-14T19:35:01,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:01,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T19:35:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:01,610 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:35:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/b5613746c97642ec8836450609f63d7b is 50, key is test_row_0/A:col10/1731612900524/Put/seqid=0 2024-11-14T19:35:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T19:35:01,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:01,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741861_1037 (size=12151) 2024-11-14T19:35:01,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612961849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612961849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612961845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612961860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612961862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612961963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612961973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612961975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612961984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:01,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612961985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,141 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/b5613746c97642ec8836450609f63d7b 2024-11-14T19:35:02,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/77f07349278844e58f606cb965c13942 is 50, key is test_row_0/B:col10/1731612900524/Put/seqid=0 2024-11-14T19:35:02,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612962183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612962194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,204 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612962195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612962194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612962204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741862_1038 (size=12151) 2024-11-14T19:35:02,247 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/77f07349278844e58f606cb965c13942 2024-11-14T19:35:02,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e1ec43a604bc473ab0b6d6845f018880 is 50, key is test_row_0/C:col10/1731612900524/Put/seqid=0 2024-11-14T19:35:02,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741863_1039 (size=12151) 2024-11-14T19:35:02,321 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e1ec43a604bc473ab0b6d6845f018880 2024-11-14T19:35:02,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/b5613746c97642ec8836450609f63d7b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b5613746c97642ec8836450609f63d7b 2024-11-14T19:35:02,366 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b5613746c97642ec8836450609f63d7b, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T19:35:02,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/77f07349278844e58f606cb965c13942 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/77f07349278844e58f606cb965c13942 2024-11-14T19:35:02,395 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/77f07349278844e58f606cb965c13942, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T19:35:02,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e1ec43a604bc473ab0b6d6845f018880 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e1ec43a604bc473ab0b6d6845f018880 2024-11-14T19:35:02,424 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e1ec43a604bc473ab0b6d6845f018880, entries=150, sequenceid=257, filesize=11.9 K 2024-11-14T19:35:02,427 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 21b827b5bf7b423416191650584b91ae in 817ms, sequenceid=257, compaction requested=true 2024-11-14T19:35:02,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:02,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:02,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T19:35:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T19:35:02,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T19:35:02,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9140 sec 2024-11-14T19:35:02,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.9300 sec 2024-11-14T19:35:02,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:35:02,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:02,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:02,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:02,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:02,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:02,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:02,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/151081dfc2364cc3813199be18104d4e is 50, key is test_row_0/A:col10/1731612902498/Put/seqid=0 2024-11-14T19:35:02,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612962550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741864_1040 (size=14741) 2024-11-14T19:35:02,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612962559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/151081dfc2364cc3813199be18104d4e 2024-11-14T19:35:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612962562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612962570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612962576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/6061fda91b8d4dda94aaefaf7a7a16ae is 50, key is test_row_0/B:col10/1731612902498/Put/seqid=0 2024-11-14T19:35:02,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741865_1041 (size=12301) 2024-11-14T19:35:02,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612962675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612962684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612962684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612962686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612962704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612962895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612962904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612962904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612962904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:02,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612962923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:02,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/6061fda91b8d4dda94aaefaf7a7a16ae 2024-11-14T19:35:03,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/6c0f70566b2f40cf8952dd662fb423df is 50, key is test_row_0/C:col10/1731612902498/Put/seqid=0 2024-11-14T19:35:03,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741866_1042 (size=12301) 2024-11-14T19:35:03,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/6c0f70566b2f40cf8952dd662fb423df 2024-11-14T19:35:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/151081dfc2364cc3813199be18104d4e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/151081dfc2364cc3813199be18104d4e 2024-11-14T19:35:03,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/151081dfc2364cc3813199be18104d4e, entries=200, sequenceid=275, filesize=14.4 K 2024-11-14T19:35:03,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/6061fda91b8d4dda94aaefaf7a7a16ae as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6061fda91b8d4dda94aaefaf7a7a16ae 2024-11-14T19:35:03,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612963213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612963224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6061fda91b8d4dda94aaefaf7a7a16ae, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T19:35:03,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612963224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612963224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612963234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/6c0f70566b2f40cf8952dd662fb423df as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/6c0f70566b2f40cf8952dd662fb423df 2024-11-14T19:35:03,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/6c0f70566b2f40cf8952dd662fb423df, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T19:35:03,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 21b827b5bf7b423416191650584b91ae in 771ms, sequenceid=275, compaction requested=true 2024-11-14T19:35:03,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:03,273 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:03,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:03,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:03,273 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:03,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:03,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:03,276 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:03,276 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:03,276 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:03,277 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/8d83012693c049f68dc16af714f427c5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/016721c5794041c18824db5e36a1dc30, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/77f07349278844e58f606cb965c13942, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6061fda91b8d4dda94aaefaf7a7a16ae] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=47.8 K 2024-11-14T19:35:03,277 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58570 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:03,277 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:03,277 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:03,278 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/774d2424330646f1bc0edb04e7895be9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2308d1f5a42b404ab28ca9edd5065dc2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b5613746c97642ec8836450609f63d7b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/151081dfc2364cc3813199be18104d4e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=57.2 K 2024-11-14T19:35:03,278 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d83012693c049f68dc16af714f427c5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731612899459 2024-11-14T19:35:03,278 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 774d2424330646f1bc0edb04e7895be9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731612899459 2024-11-14T19:35:03,279 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 016721c5794041c18824db5e36a1dc30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731612900326 2024-11-14T19:35:03,280 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2308d1f5a42b404ab28ca9edd5065dc2, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731612899555 2024-11-14T19:35:03,280 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f07349278844e58f606cb965c13942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731612900524 2024-11-14T19:35:03,281 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5613746c97642ec8836450609f63d7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731612900524 2024-11-14T19:35:03,281 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 6061fda91b8d4dda94aaefaf7a7a16ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612901855 2024-11-14T19:35:03,286 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 151081dfc2364cc3813199be18104d4e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612901845 2024-11-14T19:35:03,337 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#30 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:03,338 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/2070d735dedd41458f032c7434ffbf27 is 50, key is test_row_0/A:col10/1731612902498/Put/seqid=0 2024-11-14T19:35:03,343 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#31 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:03,344 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/25f30c76610543d89aa94d5ad121c9af is 50, key is test_row_0/B:col10/1731612902498/Put/seqid=0 2024-11-14T19:35:03,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741868_1044 (size=12643) 2024-11-14T19:35:03,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741867_1043 (size=12643) 2024-11-14T19:35:03,422 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/2070d735dedd41458f032c7434ffbf27 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2070d735dedd41458f032c7434ffbf27 2024-11-14T19:35:03,444 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into 2070d735dedd41458f032c7434ffbf27(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:03,444 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:03,444 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=12, startTime=1731612903272; duration=0sec 2024-11-14T19:35:03,444 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:03,445 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:03,459 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:03,468 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48960 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:03,468 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:03,468 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:03,468 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e2a7fdb41cd44ce8936b06f3ec449a20, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1cad4c79100443c7a644445cc8e00be3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e1ec43a604bc473ab0b6d6845f018880, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/6c0f70566b2f40cf8952dd662fb423df] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=47.8 K 2024-11-14T19:35:03,470 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a7fdb41cd44ce8936b06f3ec449a20, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731612899459 2024-11-14T19:35:03,471 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cad4c79100443c7a644445cc8e00be3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731612900326 2024-11-14T19:35:03,472 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1ec43a604bc473ab0b6d6845f018880, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731612900524 2024-11-14T19:35:03,477 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c0f70566b2f40cf8952dd662fb423df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612901855 2024-11-14T19:35:03,517 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#32 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:03,518 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/3d69188c35614e3595eb6a4aa91f8c97 is 50, key is test_row_0/C:col10/1731612902498/Put/seqid=0 2024-11-14T19:35:03,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741869_1045 (size=12643) 2024-11-14T19:35:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T19:35:03,744 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:03,745 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-14T19:35:03,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T19:35:03,757 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:03,759 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:03,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:03,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:35:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:03,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:03,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:03,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/7a82becb0e064716bfdf90da74f51a95 is 50, key is test_row_0/A:col10/1731612903769/Put/seqid=0 2024-11-14T19:35:03,805 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/25f30c76610543d89aa94d5ad121c9af as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/25f30c76610543d89aa94d5ad121c9af 2024-11-14T19:35:03,816 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 25f30c76610543d89aa94d5ad121c9af(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:03,816 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:03,816 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=12, startTime=1731612903273; duration=0sec 2024-11-14T19:35:03,820 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:03,820 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:03,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T19:35:03,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741870_1046 (size=12301) 2024-11-14T19:35:03,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:03,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:03,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:03,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:03,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:03,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612963897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612963897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612963904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612963898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612963907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:03,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:03,984 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/3d69188c35614e3595eb6a4aa91f8c97 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3d69188c35614e3595eb6a4aa91f8c97 2024-11-14T19:35:04,015 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into 3d69188c35614e3595eb6a4aa91f8c97(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:04,015 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:04,015 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=12, startTime=1731612903273; duration=0sec 2024-11-14T19:35:04,015 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:04,015 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:04,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612964024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612964024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612964033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612964045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612964045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:04,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T19:35:04,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:04,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612964244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612964254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612964244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612964265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612964265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/7a82becb0e064716bfdf90da74f51a95 2024-11-14T19:35:04,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/dca593af4fe9448f86190cae77925b57 is 50, key is test_row_0/B:col10/1731612903769/Put/seqid=0 2024-11-14T19:35:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T19:35:04,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741871_1047 (size=12301) 2024-11-14T19:35:04,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612964575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612964584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612964584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612964584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612964585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:04,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:04,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/dca593af4fe9448f86190cae77925b57 2024-11-14T19:35:04,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/cf3f96ae9dfd4c6783ce204246a4966e is 50, key is test_row_0/C:col10/1731612903769/Put/seqid=0 2024-11-14T19:35:04,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:04,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:04,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:04,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:04,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741872_1048 (size=12301) 2024-11-14T19:35:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T19:35:04,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/cf3f96ae9dfd4c6783ce204246a4966e 2024-11-14T19:35:04,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/7a82becb0e064716bfdf90da74f51a95 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a82becb0e064716bfdf90da74f51a95 2024-11-14T19:35:04,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a82becb0e064716bfdf90da74f51a95, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T19:35:04,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/dca593af4fe9448f86190cae77925b57 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dca593af4fe9448f86190cae77925b57 2024-11-14T19:35:04,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dca593af4fe9448f86190cae77925b57, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T19:35:04,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/cf3f96ae9dfd4c6783ce204246a4966e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/cf3f96ae9dfd4c6783ce204246a4966e 2024-11-14T19:35:04,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/cf3f96ae9dfd4c6783ce204246a4966e, entries=150, sequenceid=292, filesize=12.0 K 2024-11-14T19:35:04,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 21b827b5bf7b423416191650584b91ae in 1189ms, sequenceid=292, compaction requested=false 2024-11-14T19:35:04,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:05,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T19:35:05,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:05,028 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T19:35:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:05,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:05,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/c562ab6425ef4ebebff77c2508cf5ba8 is 50, key is test_row_0/A:col10/1731612903849/Put/seqid=0 2024-11-14T19:35:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:05,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:05,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741873_1049 (size=12301) 2024-11-14T19:35:05,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612965135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612965135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612965144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612965149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612965149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612965253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612965254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612965263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612965264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612965264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612965463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612965473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612965473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612965474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612965474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,493 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/c562ab6425ef4ebebff77c2508cf5ba8 2024-11-14T19:35:05,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/dd19da9a84874a4f858f92c097a7ab4e is 50, key is test_row_0/B:col10/1731612903849/Put/seqid=0 2024-11-14T19:35:05,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741874_1050 (size=12301) 2024-11-14T19:35:05,522 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/dd19da9a84874a4f858f92c097a7ab4e 2024-11-14T19:35:05,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/265e04e8a28444df872a4f24f0a91aaa is 50, key is test_row_0/C:col10/1731612903849/Put/seqid=0 2024-11-14T19:35:05,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741875_1051 (size=12301) 2024-11-14T19:35:05,552 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/265e04e8a28444df872a4f24f0a91aaa 2024-11-14T19:35:05,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/c562ab6425ef4ebebff77c2508cf5ba8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c562ab6425ef4ebebff77c2508cf5ba8 2024-11-14T19:35:05,583 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c562ab6425ef4ebebff77c2508cf5ba8, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T19:35:05,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/dd19da9a84874a4f858f92c097a7ab4e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dd19da9a84874a4f858f92c097a7ab4e 2024-11-14T19:35:05,612 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dd19da9a84874a4f858f92c097a7ab4e, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T19:35:05,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/265e04e8a28444df872a4f24f0a91aaa as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/265e04e8a28444df872a4f24f0a91aaa 2024-11-14T19:35:05,629 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/265e04e8a28444df872a4f24f0a91aaa, entries=150, sequenceid=310, filesize=12.0 K 2024-11-14T19:35:05,630 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 21b827b5bf7b423416191650584b91ae in 602ms, sequenceid=310, compaction requested=true 2024-11-14T19:35:05,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:05,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:05,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T19:35:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T19:35:05,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T19:35:05,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8740 sec 2024-11-14T19:35:05,640 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.8920 sec 2024-11-14T19:35:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:05,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:35:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:05,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:05,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/fc3a68665e3544e8b5f66e1de50556c2 is 50, key is test_row_0/A:col10/1731612905145/Put/seqid=0 2024-11-14T19:35:05,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612965800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612965802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612965804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612965804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612965807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741876_1052 (size=14741) 2024-11-14T19:35:05,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/fc3a68665e3544e8b5f66e1de50556c2 2024-11-14T19:35:05,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/cf0dac03577743a9a8a7de678c8d07bb is 50, key is test_row_0/B:col10/1731612905145/Put/seqid=0 2024-11-14T19:35:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T19:35:05,903 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:05,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-14T19:35:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T19:35:05,908 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:05,909 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:05,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:05,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612965914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612965914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612965913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612965914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:05,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:05,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741877_1053 (size=12301) 2024-11-14T19:35:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612965923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:05,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T19:35:06,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T19:35:06,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:06,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612966133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612966134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612966133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612966134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612966134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T19:35:06,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:06,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T19:35:06,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/cf0dac03577743a9a8a7de678c8d07bb 2024-11-14T19:35:06,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e47434cd4bc84097b2ec0655e89bc88b is 50, key is test_row_0/C:col10/1731612905145/Put/seqid=0 2024-11-14T19:35:06,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741878_1054 (size=12301) 2024-11-14T19:35:06,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T19:35:06,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:06,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612966453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612966453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612966453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612966454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612966454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:06,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T19:35:06,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:06,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T19:35:06,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T19:35:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:06,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e47434cd4bc84097b2ec0655e89bc88b 2024-11-14T19:35:06,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/fc3a68665e3544e8b5f66e1de50556c2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fc3a68665e3544e8b5f66e1de50556c2 2024-11-14T19:35:06,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fc3a68665e3544e8b5f66e1de50556c2, entries=200, sequenceid=328, filesize=14.4 K 2024-11-14T19:35:06,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/cf0dac03577743a9a8a7de678c8d07bb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/cf0dac03577743a9a8a7de678c8d07bb 2024-11-14T19:35:06,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/cf0dac03577743a9a8a7de678c8d07bb, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T19:35:06,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e47434cd4bc84097b2ec0655e89bc88b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e47434cd4bc84097b2ec0655e89bc88b 2024-11-14T19:35:06,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e47434cd4bc84097b2ec0655e89bc88b, entries=150, sequenceid=328, filesize=12.0 K 2024-11-14T19:35:06,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 21b827b5bf7b423416191650584b91ae in 1020ms, sequenceid=328, compaction requested=true 2024-11-14T19:35:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:06,800 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:06,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:06,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:06,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:06,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:06,800 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:06,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:06,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:06,801 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51986 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:06,802 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:06,802 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,802 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2070d735dedd41458f032c7434ffbf27, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a82becb0e064716bfdf90da74f51a95, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c562ab6425ef4ebebff77c2508cf5ba8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fc3a68665e3544e8b5f66e1de50556c2] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=50.8 K 2024-11-14T19:35:06,802 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:06,802 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:06,802 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,803 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2070d735dedd41458f032c7434ffbf27, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612901855 2024-11-14T19:35:06,803 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/25f30c76610543d89aa94d5ad121c9af, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dca593af4fe9448f86190cae77925b57, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dd19da9a84874a4f858f92c097a7ab4e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/cf0dac03577743a9a8a7de678c8d07bb] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=48.4 K 2024-11-14T19:35:06,805 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a82becb0e064716bfdf90da74f51a95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731612902561 2024-11-14T19:35:06,805 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f30c76610543d89aa94d5ad121c9af, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612901855 2024-11-14T19:35:06,805 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c562ab6425ef4ebebff77c2508cf5ba8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731612903815 2024-11-14T19:35:06,805 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting dca593af4fe9448f86190cae77925b57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731612902561 2024-11-14T19:35:06,805 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc3a68665e3544e8b5f66e1de50556c2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731612905119 2024-11-14T19:35:06,805 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting dd19da9a84874a4f858f92c097a7ab4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731612903815 2024-11-14T19:35:06,806 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0dac03577743a9a8a7de678c8d07bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731612905133 2024-11-14T19:35:06,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T19:35:06,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,835 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T19:35:06,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:06,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:06,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:06,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:06,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:06,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:06,838 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#42 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:06,838 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/6e932301f9c642d18884197f3e1a5719 is 50, key is test_row_0/B:col10/1731612905145/Put/seqid=0 2024-11-14T19:35:06,851 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#43 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:06,854 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/c94ea7fe9d7d410dbe55559ddf67e183 is 50, key is test_row_0/A:col10/1731612905145/Put/seqid=0 2024-11-14T19:35:06,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/22768114419a4c3d9c5075a74bc8ee22 is 50, key is test_row_0/A:col10/1731612905801/Put/seqid=0 2024-11-14T19:35:06,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741879_1055 (size=12779) 2024-11-14T19:35:06,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741880_1056 (size=12779) 2024-11-14T19:35:06,928 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/c94ea7fe9d7d410dbe55559ddf67e183 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c94ea7fe9d7d410dbe55559ddf67e183 2024-11-14T19:35:06,937 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into c94ea7fe9d7d410dbe55559ddf67e183(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:06,938 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:06,938 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=12, startTime=1731612906800; duration=0sec 2024-11-14T19:35:06,938 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:06,938 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:06,938 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:06,940 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:06,940 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:06,941 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:06,941 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3d69188c35614e3595eb6a4aa91f8c97, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/cf3f96ae9dfd4c6783ce204246a4966e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/265e04e8a28444df872a4f24f0a91aaa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e47434cd4bc84097b2ec0655e89bc88b] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=48.4 K 2024-11-14T19:35:06,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741881_1057 (size=12301) 2024-11-14T19:35:06,942 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d69188c35614e3595eb6a4aa91f8c97, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612901855 2024-11-14T19:35:06,944 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf3f96ae9dfd4c6783ce204246a4966e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731612902561 2024-11-14T19:35:06,944 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/22768114419a4c3d9c5075a74bc8ee22 2024-11-14T19:35:06,944 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 265e04e8a28444df872a4f24f0a91aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731612903815 2024-11-14T19:35:06,945 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e47434cd4bc84097b2ec0655e89bc88b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731612905133 2024-11-14T19:35:06,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/b8bc1eb8977145438ae17514a9bc9acb is 50, key is test_row_0/B:col10/1731612905801/Put/seqid=0 2024-11-14T19:35:06,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:06,978 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#46 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:06,979 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/c2001cb3775541158b02fad005687e9d is 50, key is test_row_0/C:col10/1731612905145/Put/seqid=0 2024-11-14T19:35:07,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612967003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612967006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612967009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612967011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612967013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741882_1058 (size=12301) 2024-11-14T19:35:07,019 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/b8bc1eb8977145438ae17514a9bc9acb 2024-11-14T19:35:07,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741883_1059 (size=12779) 2024-11-14T19:35:07,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/d8d563d3aa0a46f2b94d1d032f8e662f is 50, key is test_row_0/C:col10/1731612905801/Put/seqid=0 2024-11-14T19:35:07,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T19:35:07,051 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/c2001cb3775541158b02fad005687e9d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c2001cb3775541158b02fad005687e9d 2024-11-14T19:35:07,061 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into c2001cb3775541158b02fad005687e9d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:07,061 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:07,061 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=12, startTime=1731612906800; duration=0sec 2024-11-14T19:35:07,061 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:07,061 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:07,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741884_1060 (size=12301) 2024-11-14T19:35:07,065 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/d8d563d3aa0a46f2b94d1d032f8e662f 2024-11-14T19:35:07,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/22768114419a4c3d9c5075a74bc8ee22 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/22768114419a4c3d9c5075a74bc8ee22 2024-11-14T19:35:07,086 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/22768114419a4c3d9c5075a74bc8ee22, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T19:35:07,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/b8bc1eb8977145438ae17514a9bc9acb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b8bc1eb8977145438ae17514a9bc9acb 2024-11-14T19:35:07,096 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b8bc1eb8977145438ae17514a9bc9acb, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T19:35:07,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/d8d563d3aa0a46f2b94d1d032f8e662f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8d563d3aa0a46f2b94d1d032f8e662f 2024-11-14T19:35:07,115 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8d563d3aa0a46f2b94d1d032f8e662f, entries=150, sequenceid=342, filesize=12.0 K 2024-11-14T19:35:07,117 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 21b827b5bf7b423416191650584b91ae in 282ms, sequenceid=342, compaction requested=false 2024-11-14T19:35:07,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:07,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:07,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T19:35:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T19:35:07,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T19:35:07,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2110 sec 2024-11-14T19:35:07,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2220 sec 2024-11-14T19:35:07,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:07,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=114.05 KB heapSize=339.89 KB 2024-11-14T19:35:07,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731612907003, max timestamp=1731612907128Num uniques 859; 2024-11-14T19:35:07,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731612907003, max timestamp=1731612907128Num uniques 859; 2024-11-14T19:35:07,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:35:07,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:35:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:07,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:07,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/bfdc516536c8471da10b60ed1bd61b08 is 50, key is test_row_0/A:col10/1731612907128/Put/seqid=0 2024-11-14T19:35:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741885_1061 (size=12301) 2024-11-14T19:35:07,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612967190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612907187, max timestamp=1731612907194Num uniques 859; 2024-11-14T19:35:07,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612907187, max timestamp=1731612907194Num uniques 859; 2024-11-14T19:35:07,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:07,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:07,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612907187, max timestamp=1731612907194Num uniques 859; 2024-11-14T19:35:07,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:07,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612967221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612967227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612967227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612967227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,282 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/6e932301f9c642d18884197f3e1a5719 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6e932301f9c642d18884197f3e1a5719 2024-11-14T19:35:07,292 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 6e932301f9c642d18884197f3e1a5719(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:07,292 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:07,293 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=12, startTime=1731612906800; duration=0sec 2024-11-14T19:35:07,293 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:07,293 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:07,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612967304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612967333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612967343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612967344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612967344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612967523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612967543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612967553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612967553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/bfdc516536c8471da10b60ed1bd61b08 2024-11-14T19:35:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612967567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/ee3e45551e8d4cabbb089e95a18d4336 is 50, key is test_row_0/B:col10/1731612907128/Put/seqid=0 2024-11-14T19:35:07,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741886_1062 (size=12301) 2024-11-14T19:35:07,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612967833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612967854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612967873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612967873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612967873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:07,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/ee3e45551e8d4cabbb089e95a18d4336 2024-11-14T19:35:08,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e00d47fdf7d344acacd9db972506f8fd is 50, key is test_row_0/C:col10/1731612907128/Put/seqid=0 2024-11-14T19:35:08,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741887_1063 (size=12301) 2024-11-14T19:35:08,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e00d47fdf7d344acacd9db972506f8fd 2024-11-14T19:35:08,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/bfdc516536c8471da10b60ed1bd61b08 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/bfdc516536c8471da10b60ed1bd61b08 2024-11-14T19:35:08,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/bfdc516536c8471da10b60ed1bd61b08, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T19:35:08,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/ee3e45551e8d4cabbb089e95a18d4336 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/ee3e45551e8d4cabbb089e95a18d4336 2024-11-14T19:35:08,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/ee3e45551e8d4cabbb089e95a18d4336, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T19:35:08,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/e00d47fdf7d344acacd9db972506f8fd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e00d47fdf7d344acacd9db972506f8fd 2024-11-14T19:35:08,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T19:35:08,053 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:08,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:08,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-14T19:35:08,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T19:35:08,057 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:08,058 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:08,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:08,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e00d47fdf7d344acacd9db972506f8fd, entries=150, sequenceid=365, filesize=12.0 K 2024-11-14T19:35:08,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~62.62 KB/64120, heapSize ~174.28 KB/178464, currentSize=127.47 KB/130530 for 21b827b5bf7b423416191650584b91ae in 933ms, sequenceid=365, compaction requested=true 2024-11-14T19:35:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:08,064 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:08,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:08,065 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:08,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:08,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:08,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:08,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:08,067 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:08,067 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:08,067 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:08,067 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:08,067 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:08,067 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:08,067 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c94ea7fe9d7d410dbe55559ddf67e183, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/22768114419a4c3d9c5075a74bc8ee22, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/bfdc516536c8471da10b60ed1bd61b08] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=36.5 K 2024-11-14T19:35:08,067 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6e932301f9c642d18884197f3e1a5719, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b8bc1eb8977145438ae17514a9bc9acb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/ee3e45551e8d4cabbb089e95a18d4336] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=36.5 K 2024-11-14T19:35:08,067 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94ea7fe9d7d410dbe55559ddf67e183, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731612905133 2024-11-14T19:35:08,067 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e932301f9c642d18884197f3e1a5719, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731612905133 2024-11-14T19:35:08,068 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b8bc1eb8977145438ae17514a9bc9acb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731612905799 2024-11-14T19:35:08,068 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22768114419a4c3d9c5075a74bc8ee22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731612905799 2024-11-14T19:35:08,068 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3e45551e8d4cabbb089e95a18d4336, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731612907009 2024-11-14T19:35:08,068 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfdc516536c8471da10b60ed1bd61b08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731612907009 2024-11-14T19:35:08,082 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:08,083 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/9b64acfb9d8445b68fa5e887fcc1d3e2 is 50, key is test_row_0/B:col10/1731612907128/Put/seqid=0 2024-11-14T19:35:08,084 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:08,084 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/fe86626ee3ec4f1dbb6b68d6f7eba124 is 50, key is test_row_0/A:col10/1731612907128/Put/seqid=0 2024-11-14T19:35:08,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741888_1064 (size=12881) 2024-11-14T19:35:08,098 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/9b64acfb9d8445b68fa5e887fcc1d3e2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9b64acfb9d8445b68fa5e887fcc1d3e2 2024-11-14T19:35:08,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741889_1065 (size=12881) 2024-11-14T19:35:08,110 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 9b64acfb9d8445b68fa5e887fcc1d3e2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:08,110 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:08,110 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=13, startTime=1731612908065; duration=0sec 2024-11-14T19:35:08,110 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:08,110 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:08,110 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:08,112 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:08,112 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:08,112 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:08,113 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c2001cb3775541158b02fad005687e9d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8d563d3aa0a46f2b94d1d032f8e662f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e00d47fdf7d344acacd9db972506f8fd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=36.5 K 2024-11-14T19:35:08,113 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c2001cb3775541158b02fad005687e9d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731612905133 2024-11-14T19:35:08,114 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d563d3aa0a46f2b94d1d032f8e662f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731612905799 2024-11-14T19:35:08,115 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting e00d47fdf7d344acacd9db972506f8fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731612907009 2024-11-14T19:35:08,124 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:08,125 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/c0838421e38a4467984b4196abbe316e is 50, key is test_row_0/C:col10/1731612907128/Put/seqid=0 2024-11-14T19:35:08,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741890_1066 (size=12881) 2024-11-14T19:35:08,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T19:35:08,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-14T19:35:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:08,212 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-14T19:35:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:08,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:08,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/203fd011f2884ecca582e4d9f18ecf06 is 50, key is test_row_0/A:col10/1731612907213/Put/seqid=0 2024-11-14T19:35:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741891_1067 (size=12301) 2024-11-14T19:35:08,224 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/203fd011f2884ecca582e4d9f18ecf06 2024-11-14T19:35:08,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/627492b26ef246d584d02eb0e9d32510 is 50, key is test_row_0/B:col10/1731612907213/Put/seqid=0 2024-11-14T19:35:08,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741892_1068 (size=12301) 2024-11-14T19:35:08,252 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/627492b26ef246d584d02eb0e9d32510 2024-11-14T19:35:08,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/18c645967e7e40718ade6f3e11d8069d is 50, key is test_row_0/C:col10/1731612907213/Put/seqid=0 2024-11-14T19:35:08,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741893_1069 (size=12301) 2024-11-14T19:35:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:08,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612968368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612968368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T19:35:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612968383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612968383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612968394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612968484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612968484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,501 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/fe86626ee3ec4f1dbb6b68d6f7eba124 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fe86626ee3ec4f1dbb6b68d6f7eba124 2024-11-14T19:35:08,508 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into fe86626ee3ec4f1dbb6b68d6f7eba124(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:08,509 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:08,509 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=13, startTime=1731612908064; duration=0sec 2024-11-14T19:35:08,509 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:08,509 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:08,552 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/c0838421e38a4467984b4196abbe316e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c0838421e38a4467984b4196abbe316e 2024-11-14T19:35:08,559 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into c0838421e38a4467984b4196abbe316e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:08,560 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:08,560 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=13, startTime=1731612908065; duration=0sec 2024-11-14T19:35:08,560 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:08,560 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:08,674 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/18c645967e7e40718ade6f3e11d8069d 2024-11-14T19:35:08,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/203fd011f2884ecca582e4d9f18ecf06 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/203fd011f2884ecca582e4d9f18ecf06 2024-11-14T19:35:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T19:35:08,687 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/203fd011f2884ecca582e4d9f18ecf06, entries=150, sequenceid=401, filesize=12.0 K 2024-11-14T19:35:08,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/627492b26ef246d584d02eb0e9d32510 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/627492b26ef246d584d02eb0e9d32510 2024-11-14T19:35:08,700 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/627492b26ef246d584d02eb0e9d32510, entries=150, sequenceid=401, filesize=12.0 K 2024-11-14T19:35:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612968693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612968693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/18c645967e7e40718ade6f3e11d8069d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/18c645967e7e40718ade6f3e11d8069d 2024-11-14T19:35:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:08,714 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/18c645967e7e40718ade6f3e11d8069d, entries=150, sequenceid=401, filesize=12.0 K 2024-11-14T19:35:08,717 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 21b827b5bf7b423416191650584b91ae in 505ms, sequenceid=401, compaction requested=false 2024-11-14T19:35:08,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:08,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:08,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-14T19:35:08,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-14T19:35:08,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-14T19:35:08,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 661 msec 2024-11-14T19:35:08,725 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 669 msec 2024-11-14T19:35:09,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:09,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:35:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/99bfd5abb51343eb835d041f3d0e1e0f is 50, key is test_row_0/A:col10/1731612909014/Put/seqid=0 2024-11-14T19:35:09,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741894_1070 (size=9857) 2024-11-14T19:35:09,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/99bfd5abb51343eb835d041f3d0e1e0f 2024-11-14T19:35:09,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/a7883d937eb74a9aa419d145dd7c71b1 is 50, key is test_row_0/B:col10/1731612909014/Put/seqid=0 2024-11-14T19:35:09,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741895_1071 (size=9857) 2024-11-14T19:35:09,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/a7883d937eb74a9aa419d145dd7c71b1 2024-11-14T19:35:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612969141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612969140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/910262081977409abe5a92670abe376d is 50, key is test_row_0/C:col10/1731612909014/Put/seqid=0 2024-11-14T19:35:09,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741896_1072 (size=9857) 2024-11-14T19:35:09,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/910262081977409abe5a92670abe376d 2024-11-14T19:35:09,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/99bfd5abb51343eb835d041f3d0e1e0f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/99bfd5abb51343eb835d041f3d0e1e0f 2024-11-14T19:35:09,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/99bfd5abb51343eb835d041f3d0e1e0f, entries=100, sequenceid=416, filesize=9.6 K 2024-11-14T19:35:09,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/a7883d937eb74a9aa419d145dd7c71b1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7883d937eb74a9aa419d145dd7c71b1 2024-11-14T19:35:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-14T19:35:09,194 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:09,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:09,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7883d937eb74a9aa419d145dd7c71b1, entries=100, sequenceid=416, filesize=9.6 K 2024-11-14T19:35:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-14T19:35:09,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/910262081977409abe5a92670abe376d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/910262081977409abe5a92670abe376d 2024-11-14T19:35:09,199 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:09,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T19:35:09,200 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:09,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:09,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/910262081977409abe5a92670abe376d, entries=100, sequenceid=416, filesize=9.6 K 2024-11-14T19:35:09,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 21b827b5bf7b423416191650584b91ae in 124ms, sequenceid=416, compaction requested=true 2024-11-14T19:35:09,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:09,219 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:09,220 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:09,221 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:09,221 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:09,221 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:09,221 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fe86626ee3ec4f1dbb6b68d6f7eba124, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/203fd011f2884ecca582e4d9f18ecf06, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/99bfd5abb51343eb835d041f3d0e1e0f] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=34.2 K 2024-11-14T19:35:09,222 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe86626ee3ec4f1dbb6b68d6f7eba124, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731612907009 2024-11-14T19:35:09,222 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:09,222 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:09,222 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:09,222 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9b64acfb9d8445b68fa5e887fcc1d3e2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/627492b26ef246d584d02eb0e9d32510, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7883d937eb74a9aa419d145dd7c71b1] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=34.2 K 2024-11-14T19:35:09,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:09,223 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203fd011f2884ecca582e4d9f18ecf06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731612907213 2024-11-14T19:35:09,223 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b64acfb9d8445b68fa5e887fcc1d3e2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731612907009 2024-11-14T19:35:09,223 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 627492b26ef246d584d02eb0e9d32510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731612907213 2024-11-14T19:35:09,223 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99bfd5abb51343eb835d041f3d0e1e0f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731612909014 2024-11-14T19:35:09,224 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a7883d937eb74a9aa419d145dd7c71b1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731612909014 2024-11-14T19:35:09,235 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:09,236 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/980436ee69b94518ace5f1ae5ff63c48 is 50, key is test_row_0/B:col10/1731612909014/Put/seqid=0 2024-11-14T19:35:09,245 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:09,245 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/8ae8b0c3b510438f9b12cfba9777fedd is 50, key is test_row_0/A:col10/1731612909014/Put/seqid=0 2024-11-14T19:35:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:09,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:35:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:09,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741897_1073 (size=12983) 2024-11-14T19:35:09,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612969279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/68f4037872f7419a9ec9e0d27e1056b3 is 50, key is test_row_0/A:col10/1731612909254/Put/seqid=0 2024-11-14T19:35:09,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612969281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741898_1074 (size=12983) 2024-11-14T19:35:09,301 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/8ae8b0c3b510438f9b12cfba9777fedd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8ae8b0c3b510438f9b12cfba9777fedd 2024-11-14T19:35:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T19:35:09,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741899_1075 (size=12301) 2024-11-14T19:35:09,315 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into 8ae8b0c3b510438f9b12cfba9777fedd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:09,315 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:09,315 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=13, startTime=1731612909219; duration=0sec 2024-11-14T19:35:09,315 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:09,316 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:09,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/68f4037872f7419a9ec9e0d27e1056b3 2024-11-14T19:35:09,316 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:09,319 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:09,319 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:09,319 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:09,319 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c0838421e38a4467984b4196abbe316e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/18c645967e7e40718ade6f3e11d8069d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/910262081977409abe5a92670abe376d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=34.2 K 2024-11-14T19:35:09,321 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0838421e38a4467984b4196abbe316e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731612907009 2024-11-14T19:35:09,325 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c645967e7e40718ade6f3e11d8069d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731612907213 2024-11-14T19:35:09,325 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 910262081977409abe5a92670abe376d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731612909014 2024-11-14T19:35:09,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/b9b3f4eeb12e48a3a6828617a90b9032 is 50, key is test_row_0/B:col10/1731612909254/Put/seqid=0 2024-11-14T19:35:09,344 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:09,345 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/60612e424ece42c3bb46243f5deadf42 is 50, key is test_row_0/C:col10/1731612909014/Put/seqid=0 2024-11-14T19:35:09,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T19:35:09,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:09,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:09,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:09,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:09,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:09,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741900_1076 (size=12301) 2024-11-14T19:35:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:09,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/b9b3f4eeb12e48a3a6828617a90b9032 2024-11-14T19:35:09,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741901_1077 (size=12983) 2024-11-14T19:35:09,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/4de7c1272325489298f986684955b400 is 50, key is test_row_0/C:col10/1731612909254/Put/seqid=0 2024-11-14T19:35:09,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741902_1078 (size=12301) 2024-11-14T19:35:09,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612969393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612969393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/4de7c1272325489298f986684955b400 2024-11-14T19:35:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612969393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,400 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 21b827b5bf7b423416191650584b91ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612969403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612969403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,404 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 21b827b5bf7b423416191650584b91ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:09,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,405 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 21b827b5bf7b423416191650584b91ae of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/68f4037872f7419a9ec9e0d27e1056b3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/68f4037872f7419a9ec9e0d27e1056b3 2024-11-14T19:35:09,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/68f4037872f7419a9ec9e0d27e1056b3, entries=150, sequenceid=436, filesize=12.0 K 2024-11-14T19:35:09,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/b9b3f4eeb12e48a3a6828617a90b9032 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b9b3f4eeb12e48a3a6828617a90b9032 2024-11-14T19:35:09,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b9b3f4eeb12e48a3a6828617a90b9032, entries=150, sequenceid=436, filesize=12.0 K 2024-11-14T19:35:09,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/4de7c1272325489298f986684955b400 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4de7c1272325489298f986684955b400 2024-11-14T19:35:09,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4de7c1272325489298f986684955b400, entries=150, sequenceid=436, filesize=12.0 K 2024-11-14T19:35:09,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 21b827b5bf7b423416191650584b91ae in 191ms, sequenceid=436, compaction requested=false 2024-11-14T19:35:09,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:09,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-14T19:35:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:09,511 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:35:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:09,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T19:35:09,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/1fb814cc2fa1415d9487430cc468ff6f is 50, key is test_row_0/A:col10/1731612909277/Put/seqid=0 2024-11-14T19:35:09,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741903_1079 (size=12301) 2024-11-14T19:35:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:09,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:09,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731612909611, max timestamp=1731612909656Num uniques 859; 2024-11-14T19:35:09,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731612909611, max timestamp=1731612909656Num uniques 859; 2024-11-14T19:35:09,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:09,658 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:09,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612909611, max timestamp=1731612909656Num uniques 859; 2024-11-14T19:35:09,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:09,687 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/980436ee69b94518ace5f1ae5ff63c48 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/980436ee69b94518ace5f1ae5ff63c48 2024-11-14T19:35:09,698 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 980436ee69b94518ace5f1ae5ff63c48(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:09,698 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:09,698 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=13, startTime=1731612909220; duration=0sec 2024-11-14T19:35:09,698 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:09,698 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:09,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612969708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612969712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,785 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/60612e424ece42c3bb46243f5deadf42 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/60612e424ece42c3bb46243f5deadf42 2024-11-14T19:35:09,791 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into 60612e424ece42c3bb46243f5deadf42(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:09,792 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:09,792 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=13, startTime=1731612909220; duration=0sec 2024-11-14T19:35:09,792 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:09,792 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T19:35:09,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612969823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612969823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:09,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:09,936 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/1fb814cc2fa1415d9487430cc468ff6f 2024-11-14T19:35:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/fc86e66065124d22acf3c97f7df8aed6 is 50, key is test_row_0/B:col10/1731612909277/Put/seqid=0 2024-11-14T19:35:09,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741904_1080 (size=12301) 2024-11-14T19:35:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612970033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:10,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612970033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T19:35:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612970343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:10,351 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/fc86e66065124d22acf3c97f7df8aed6 2024-11-14T19:35:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612970353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/d8b39ad714eb433294ad86c3a0fd9217 is 50, key is test_row_0/C:col10/1731612909277/Put/seqid=0 2024-11-14T19:35:10,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741905_1081 (size=12301) 2024-11-14T19:35:10,766 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/d8b39ad714eb433294ad86c3a0fd9217 2024-11-14T19:35:10,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/1fb814cc2fa1415d9487430cc468ff6f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1fb814cc2fa1415d9487430cc468ff6f 2024-11-14T19:35:10,778 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1fb814cc2fa1415d9487430cc468ff6f, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T19:35:10,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/fc86e66065124d22acf3c97f7df8aed6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/fc86e66065124d22acf3c97f7df8aed6 2024-11-14T19:35:10,785 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/fc86e66065124d22acf3c97f7df8aed6, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T19:35:10,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/d8b39ad714eb433294ad86c3a0fd9217 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8b39ad714eb433294ad86c3a0fd9217 2024-11-14T19:35:10,792 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8b39ad714eb433294ad86c3a0fd9217, entries=150, sequenceid=449, filesize=12.0 K 2024-11-14T19:35:10,793 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 21b827b5bf7b423416191650584b91ae in 1281ms, sequenceid=449, compaction requested=true 2024-11-14T19:35:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:10,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-14T19:35:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-14T19:35:10,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-14T19:35:10,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5930 sec 2024-11-14T19:35:10,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6020 sec 2024-11-14T19:35:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:10,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-14T19:35:10,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:10,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:10,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:10,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:10,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:10,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:10,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/e38e37de00d34813b0c5da2528a4d930 is 50, key is test_row_0/A:col10/1731612909711/Put/seqid=0 2024-11-14T19:35:10,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741906_1082 (size=14741) 2024-11-14T19:35:10,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612970876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:10,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612970885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:10,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612970993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:10,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:10,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612970993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612971203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612971203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/e38e37de00d34813b0c5da2528a4d930 2024-11-14T19:35:11,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/9ebad643f218471bb9ba68a7581c441d is 50, key is test_row_0/B:col10/1731612909711/Put/seqid=0 2024-11-14T19:35:11,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741907_1083 (size=12301) 2024-11-14T19:35:11,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/9ebad643f218471bb9ba68a7581c441d 2024-11-14T19:35:11,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/dd015a437069433e9d3a1a185166723b is 50, key is test_row_0/C:col10/1731612909711/Put/seqid=0 2024-11-14T19:35:11,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741908_1084 (size=12301) 2024-11-14T19:35:11,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-14T19:35:11,343 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:11,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:11,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-14T19:35:11,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:11,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T19:35:11,346 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:11,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:11,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56402 deadline: 1731612971414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56472 deadline: 1731612971414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:11,416 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 21b827b5bf7b423416191650584b91ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56436 deadline: 1731612971414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,417 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 21b827b5bf7b423416191650584b91ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,417 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 21b827b5bf7b423416191650584b91ae of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T19:35:11,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T19:35:11,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:11,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:11,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:11,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612971514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612971514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:11,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:11,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T19:35:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:11,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:11,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T19:35:11,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/dd015a437069433e9d3a1a185166723b 2024-11-14T19:35:11,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/e38e37de00d34813b0c5da2528a4d930 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e38e37de00d34813b0c5da2528a4d930 2024-11-14T19:35:11,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e38e37de00d34813b0c5da2528a4d930, entries=200, sequenceid=485, filesize=14.4 K 2024-11-14T19:35:11,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/9ebad643f218471bb9ba68a7581c441d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9ebad643f218471bb9ba68a7581c441d 2024-11-14T19:35:11,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9ebad643f218471bb9ba68a7581c441d, entries=150, sequenceid=485, filesize=12.0 K 2024-11-14T19:35:11,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/dd015a437069433e9d3a1a185166723b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/dd015a437069433e9d3a1a185166723b 2024-11-14T19:35:11,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/dd015a437069433e9d3a1a185166723b, entries=150, sequenceid=485, filesize=12.0 K 2024-11-14T19:35:11,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 21b827b5bf7b423416191650584b91ae in 885ms, sequenceid=485, compaction requested=true 2024-11-14T19:35:11,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:11,741 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:11,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:11,743 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:11,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:11,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:11,746 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:11,746 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:11,746 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,746 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8ae8b0c3b510438f9b12cfba9777fedd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/68f4037872f7419a9ec9e0d27e1056b3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1fb814cc2fa1415d9487430cc468ff6f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e38e37de00d34813b0c5da2528a4d930] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=51.1 K 2024-11-14T19:35:11,747 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae8b0c3b510438f9b12cfba9777fedd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731612907224 2024-11-14T19:35:11,747 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:11,747 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:11,747 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,747 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/980436ee69b94518ace5f1ae5ff63c48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b9b3f4eeb12e48a3a6828617a90b9032, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/fc86e66065124d22acf3c97f7df8aed6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9ebad643f218471bb9ba68a7581c441d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=48.7 K 2024-11-14T19:35:11,748 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68f4037872f7419a9ec9e0d27e1056b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731612909130 2024-11-14T19:35:11,748 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 980436ee69b94518ace5f1ae5ff63c48, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731612907224 2024-11-14T19:35:11,748 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fb814cc2fa1415d9487430cc468ff6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731612909272 2024-11-14T19:35:11,748 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b9b3f4eeb12e48a3a6828617a90b9032, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731612909130 2024-11-14T19:35:11,749 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e38e37de00d34813b0c5da2528a4d930, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731612909686 2024-11-14T19:35:11,749 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting fc86e66065124d22acf3c97f7df8aed6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731612909272 2024-11-14T19:35:11,750 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ebad643f218471bb9ba68a7581c441d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731612909686 2024-11-14T19:35:11,781 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:11,782 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/9bc9b51fa56149638a5875af095e8efc is 50, key is test_row_0/A:col10/1731612909711/Put/seqid=0 2024-11-14T19:35:11,789 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:11,790 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d5129f9d9f664905b2158b526ffdf5dd is 50, key is test_row_0/B:col10/1731612909711/Put/seqid=0 2024-11-14T19:35:11,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-14T19:35:11,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,808 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T19:35:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:11,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:11,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741909_1085 (size=13119) 2024-11-14T19:35:11,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741910_1086 (size=13119) 2024-11-14T19:35:11,828 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/9bc9b51fa56149638a5875af095e8efc as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/9bc9b51fa56149638a5875af095e8efc 2024-11-14T19:35:11,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/d16d090797064869b75887860ba44556 is 50, key is test_row_0/A:col10/1731612910883/Put/seqid=0 2024-11-14T19:35:11,840 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d5129f9d9f664905b2158b526ffdf5dd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d5129f9d9f664905b2158b526ffdf5dd 2024-11-14T19:35:11,844 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into 9bc9b51fa56149638a5875af095e8efc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:11,844 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:11,844 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=12, startTime=1731612911741; duration=0sec 2024-11-14T19:35:11,844 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:11,844 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:11,845 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:11,850 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:11,850 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:11,850 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into d5129f9d9f664905b2158b526ffdf5dd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:11,850 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:11,850 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:11,850 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=12, startTime=1731612911743; duration=0sec 2024-11-14T19:35:11,850 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/60612e424ece42c3bb46243f5deadf42, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4de7c1272325489298f986684955b400, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8b39ad714eb433294ad86c3a0fd9217, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/dd015a437069433e9d3a1a185166723b] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=48.7 K 2024-11-14T19:35:11,851 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:11,851 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:11,851 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60612e424ece42c3bb46243f5deadf42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731612907224 2024-11-14T19:35:11,851 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4de7c1272325489298f986684955b400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731612909130 2024-11-14T19:35:11,852 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8b39ad714eb433294ad86c3a0fd9217, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731612909272 2024-11-14T19:35:11,853 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd015a437069433e9d3a1a185166723b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731612909686 2024-11-14T19:35:11,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741911_1087 (size=12301) 2024-11-14T19:35:11,874 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/d16d090797064869b75887860ba44556 2024-11-14T19:35:11,879 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#75 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:11,880 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/b8ed4fa41360457ca3f4d764cbdaf3a9 is 50, key is test_row_0/C:col10/1731612909711/Put/seqid=0 2024-11-14T19:35:11,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/f4e2a872b91f4ebb81fadbbd90849a0c is 50, key is test_row_0/B:col10/1731612910883/Put/seqid=0 2024-11-14T19:35:11,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741912_1088 (size=13119) 2024-11-14T19:35:11,929 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/b8ed4fa41360457ca3f4d764cbdaf3a9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/b8ed4fa41360457ca3f4d764cbdaf3a9 2024-11-14T19:35:11,937 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into b8ed4fa41360457ca3f4d764cbdaf3a9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:11,937 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:11,937 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=12, startTime=1731612911743; duration=0sec 2024-11-14T19:35:11,937 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:11,937 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:11,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741913_1089 (size=12301) 2024-11-14T19:35:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T19:35:12,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:12,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:12,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612912042, max timestamp=1731612912049Num uniques 859; 2024-11-14T19:35:12,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612912042, max timestamp=1731612912049Num uniques 859; 2024-11-14T19:35:12,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612912042, max timestamp=1731612912049Num uniques 859; 2024-11-14T19:35:12,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:12,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:12,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:12,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612972082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612972085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612972193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612972193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,340 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/f4e2a872b91f4ebb81fadbbd90849a0c 2024-11-14T19:35:12,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/ad864453eb924de39835d6e2072c26cd is 50, key is test_row_0/C:col10/1731612910883/Put/seqid=0 2024-11-14T19:35:12,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741914_1090 (size=12301) 2024-11-14T19:35:12,354 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/ad864453eb924de39835d6e2072c26cd 2024-11-14T19:35:12,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/d16d090797064869b75887860ba44556 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d16d090797064869b75887860ba44556 2024-11-14T19:35:12,375 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d16d090797064869b75887860ba44556, entries=150, sequenceid=496, filesize=12.0 K 2024-11-14T19:35:12,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/f4e2a872b91f4ebb81fadbbd90849a0c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/f4e2a872b91f4ebb81fadbbd90849a0c 2024-11-14T19:35:12,387 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/f4e2a872b91f4ebb81fadbbd90849a0c, entries=150, sequenceid=496, filesize=12.0 K 2024-11-14T19:35:12,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/ad864453eb924de39835d6e2072c26cd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ad864453eb924de39835d6e2072c26cd 2024-11-14T19:35:12,395 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ad864453eb924de39835d6e2072c26cd, entries=150, sequenceid=496, filesize=12.0 K 2024-11-14T19:35:12,396 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 21b827b5bf7b423416191650584b91ae in 589ms, sequenceid=496, compaction requested=false 2024-11-14T19:35:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-14T19:35:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-14T19:35:12,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-14T19:35:12,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0500 sec 2024-11-14T19:35:12,401 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.0560 sec 2024-11-14T19:35:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:12,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-14T19:35:12,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:12,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:12,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:12,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:12,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:12,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:12,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/d195f1aa5859464f85587ec9e7d4dfb7 is 50, key is test_row_0/A:col10/1731612912079/Put/seqid=0 2024-11-14T19:35:12,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741915_1091 (size=17181) 2024-11-14T19:35:12,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612972431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612972430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-14T19:35:12,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:12,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-14T19:35:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T19:35:12,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:12,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:12,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612972543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612972543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T19:35:12,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T19:35:12,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:12,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:12,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612972753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612972753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:12,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T19:35:12,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:12,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,792 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:12,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T19:35:12,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/d195f1aa5859464f85587ec9e7d4dfb7 2024-11-14T19:35:12,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/3a897ca4ef734f2fbd2aaf917486ad1e is 50, key is test_row_0/B:col10/1731612912079/Put/seqid=0 2024-11-14T19:35:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741916_1092 (size=12301) 2024-11-14T19:35:12,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/3a897ca4ef734f2fbd2aaf917486ad1e 2024-11-14T19:35:12,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/77b009860a2e4ad5a91f3059fb3c3645 is 50, key is test_row_0/C:col10/1731612912079/Put/seqid=0 2024-11-14T19:35:12,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741917_1093 (size=12301) 2024-11-14T19:35:12,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/77b009860a2e4ad5a91f3059fb3c3645 2024-11-14T19:35:12,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/d195f1aa5859464f85587ec9e7d4dfb7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d195f1aa5859464f85587ec9e7d4dfb7 2024-11-14T19:35:12,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d195f1aa5859464f85587ec9e7d4dfb7, entries=250, sequenceid=535, filesize=16.8 K 2024-11-14T19:35:12,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/3a897ca4ef734f2fbd2aaf917486ad1e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3a897ca4ef734f2fbd2aaf917486ad1e 2024-11-14T19:35:12,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3a897ca4ef734f2fbd2aaf917486ad1e, entries=150, sequenceid=535, filesize=12.0 K 2024-11-14T19:35:12,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/77b009860a2e4ad5a91f3059fb3c3645 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/77b009860a2e4ad5a91f3059fb3c3645 2024-11-14T19:35:12,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/77b009860a2e4ad5a91f3059fb3c3645, entries=150, sequenceid=535, filesize=12.0 K 2024-11-14T19:35:12,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 21b827b5bf7b423416191650584b91ae in 533ms, sequenceid=535, compaction requested=true 2024-11-14T19:35:12,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:12,940 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:12,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:12,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:12,941 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:12,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:12,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:12,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:12,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:12,943 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42601 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:12,943 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:12,944 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:12,944 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,944 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:12,944 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/9bc9b51fa56149638a5875af095e8efc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d16d090797064869b75887860ba44556, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d195f1aa5859464f85587ec9e7d4dfb7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=41.6 K 2024-11-14T19:35:12,944 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,944 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d5129f9d9f664905b2158b526ffdf5dd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/f4e2a872b91f4ebb81fadbbd90849a0c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3a897ca4ef734f2fbd2aaf917486ad1e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=36.8 K 2024-11-14T19:35:12,944 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bc9b51fa56149638a5875af095e8efc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731612909686 2024-11-14T19:35:12,944 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting d5129f9d9f664905b2158b526ffdf5dd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731612909686 2024-11-14T19:35:12,945 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f4e2a872b91f4ebb81fadbbd90849a0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731612910871 2024-11-14T19:35:12,945 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d16d090797064869b75887860ba44556, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731612910871 2024-11-14T19:35:12,945 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a897ca4ef734f2fbd2aaf917486ad1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731612912079 2024-11-14T19:35:12,946 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d195f1aa5859464f85587ec9e7d4dfb7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731612912079 2024-11-14T19:35:12,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-14T19:35:12,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:12,947 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-14T19:35:12,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:12,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:12,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:12,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:12,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/a5e4406a2c0b43f8b4f6a7b5cd26c941 is 50, key is test_row_1/A:col10/1731612912429/Put/seqid=0 2024-11-14T19:35:12,973 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:12,974 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/3f38357279f347038ae0752162c66a76 is 50, key is test_row_0/B:col10/1731612912079/Put/seqid=0 2024-11-14T19:35:12,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741918_1094 (size=9857) 2024-11-14T19:35:12,996 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/a5e4406a2c0b43f8b4f6a7b5cd26c941 2024-11-14T19:35:13,009 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:13,010 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/e52adad4f747439f88d89ef35a7f980d is 50, key is test_row_0/A:col10/1731612912079/Put/seqid=0 2024-11-14T19:35:13,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d2ab8cabfcfc44c39cf63d4aafa45a92 is 50, key is test_row_1/B:col10/1731612912429/Put/seqid=0 2024-11-14T19:35:13,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741919_1095 (size=13221) 2024-11-14T19:35:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741920_1096 (size=13221) 2024-11-14T19:35:13,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741921_1097 (size=9857) 2024-11-14T19:35:13,059 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d2ab8cabfcfc44c39cf63d4aafa45a92 2024-11-14T19:35:13,063 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/3f38357279f347038ae0752162c66a76 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3f38357279f347038ae0752162c66a76 2024-11-14T19:35:13,071 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/e52adad4f747439f88d89ef35a7f980d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e52adad4f747439f88d89ef35a7f980d 2024-11-14T19:35:13,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/7a67244f3a7c4027bbbcc5d776c56a4c is 50, key is test_row_1/C:col10/1731612912429/Put/seqid=0 2024-11-14T19:35:13,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741922_1098 (size=9857) 2024-11-14T19:35:13,089 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into e52adad4f747439f88d89ef35a7f980d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:13,089 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 3f38357279f347038ae0752162c66a76(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:13,089 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:13,089 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:13,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:13,089 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=13, startTime=1731612912941; duration=0sec 2024-11-14T19:35:13,089 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=13, startTime=1731612912940; duration=0sec 2024-11-14T19:35:13,090 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:13,090 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:13,090 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:13,090 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:13,090 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:13,092 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:13,092 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:13,092 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:13,093 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/b8ed4fa41360457ca3f4d764cbdaf3a9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ad864453eb924de39835d6e2072c26cd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/77b009860a2e4ad5a91f3059fb3c3645] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=36.8 K 2024-11-14T19:35:13,093 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b8ed4fa41360457ca3f4d764cbdaf3a9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731612909686 2024-11-14T19:35:13,094 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ad864453eb924de39835d6e2072c26cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731612910871 2024-11-14T19:35:13,094 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 77b009860a2e4ad5a91f3059fb3c3645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731612912079 2024-11-14T19:35:13,108 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:13,109 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/28457c92767b45899b103a65425c19be is 50, key is test_row_0/C:col10/1731612912079/Put/seqid=0 2024-11-14T19:35:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T19:35:13,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741923_1099 (size=13221) 2024-11-14T19:35:13,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612913131, max timestamp=1731612913153Num uniques 859; 2024-11-14T19:35:13,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612913131, max timestamp=1731612913153Num uniques 859; 2024-11-14T19:35:13,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:13,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:13,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731612913131, max timestamp=1731612913153Num uniques 859; 2024-11-14T19:35:13,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:35:13,163 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/28457c92767b45899b103a65425c19be as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/28457c92767b45899b103a65425c19be 2024-11-14T19:35:13,172 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into 28457c92767b45899b103a65425c19be(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:13,172 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:13,172 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=13, startTime=1731612912941; duration=0sec 2024-11-14T19:35:13,172 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:13,172 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:13,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612973207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612973207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612973313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612973323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,489 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/7a67244f3a7c4027bbbcc5d776c56a4c 2024-11-14T19:35:13,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/a5e4406a2c0b43f8b4f6a7b5cd26c941 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/a5e4406a2c0b43f8b4f6a7b5cd26c941 2024-11-14T19:35:13,507 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/a5e4406a2c0b43f8b4f6a7b5cd26c941, entries=100, sequenceid=544, filesize=9.6 K 2024-11-14T19:35:13,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d2ab8cabfcfc44c39cf63d4aafa45a92 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d2ab8cabfcfc44c39cf63d4aafa45a92 2024-11-14T19:35:13,514 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d2ab8cabfcfc44c39cf63d4aafa45a92, entries=100, sequenceid=544, filesize=9.6 K 2024-11-14T19:35:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/7a67244f3a7c4027bbbcc5d776c56a4c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/7a67244f3a7c4027bbbcc5d776c56a4c 2024-11-14T19:35:13,524 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/7a67244f3a7c4027bbbcc5d776c56a4c, entries=100, sequenceid=544, filesize=9.6 K 2024-11-14T19:35:13,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612973523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,525 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 21b827b5bf7b423416191650584b91ae in 578ms, sequenceid=544, compaction requested=false 2024-11-14T19:35:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:13,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:13,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-14T19:35:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-14T19:35:13,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-14T19:35:13,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0410 sec 2024-11-14T19:35:13,530 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.0460 sec 2024-11-14T19:35:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:13,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-14T19:35:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:13,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:13,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/7a6107b094ed494287396537dd5d51a1 is 50, key is test_row_0/A:col10/1731612913533/Put/seqid=0 2024-11-14T19:35:13,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741924_1100 (size=14741) 2024-11-14T19:35:13,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612973561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-14T19:35:13,623 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:13,624 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-14T19:35:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T19:35:13,626 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:13,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:13,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:13,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612973673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T19:35:13,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T19:35:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:13,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:13,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612973833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:13,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612973883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:13,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T19:35:13,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:13,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:13,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:13,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:13,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T19:35:13,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/7a6107b094ed494287396537dd5d51a1 2024-11-14T19:35:13,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d3d9af0d31434734b9e73f1574a9f09a is 50, key is test_row_0/B:col10/1731612913533/Put/seqid=0 2024-11-14T19:35:14,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741925_1101 (size=12301) 2024-11-14T19:35:14,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d3d9af0d31434734b9e73f1574a9f09a 2024-11-14T19:35:14,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/69b9f779d773445ca22a3f19b0e50bad is 50, key is test_row_0/C:col10/1731612913533/Put/seqid=0 2024-11-14T19:35:14,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741926_1102 (size=12301) 2024-11-14T19:35:14,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T19:35:14,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:14,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612974203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:14,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:14,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T19:35:14,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:14,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T19:35:14,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:14,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56406 deadline: 1731612974343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:14,345 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:14,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T19:35:14,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:14,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:14,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/69b9f779d773445ca22a3f19b0e50bad 2024-11-14T19:35:14,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/7a6107b094ed494287396537dd5d51a1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a6107b094ed494287396537dd5d51a1 2024-11-14T19:35:14,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a6107b094ed494287396537dd5d51a1, entries=200, sequenceid=584, filesize=14.4 K 2024-11-14T19:35:14,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/d3d9af0d31434734b9e73f1574a9f09a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d3d9af0d31434734b9e73f1574a9f09a 2024-11-14T19:35:14,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d3d9af0d31434734b9e73f1574a9f09a, entries=150, sequenceid=584, filesize=12.0 K 2024-11-14T19:35:14,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/69b9f779d773445ca22a3f19b0e50bad as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/69b9f779d773445ca22a3f19b0e50bad 2024-11-14T19:35:14,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/69b9f779d773445ca22a3f19b0e50bad, entries=150, sequenceid=584, filesize=12.0 K 2024-11-14T19:35:14,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 21b827b5bf7b423416191650584b91ae in 976ms, sequenceid=584, compaction requested=true 2024-11-14T19:35:14,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:14,511 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:14,512 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37819 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:14,512 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/A is initiating minor compaction (all files) 2024-11-14T19:35:14,513 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/A in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,513 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e52adad4f747439f88d89ef35a7f980d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/a5e4406a2c0b43f8b4f6a7b5cd26c941, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a6107b094ed494287396537dd5d51a1] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=36.9 K 2024-11-14T19:35:14,513 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52adad4f747439f88d89ef35a7f980d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731612912079 2024-11-14T19:35:14,514 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5e4406a2c0b43f8b4f6a7b5cd26c941, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731612912428 2024-11-14T19:35:14,514 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a6107b094ed494287396537dd5d51a1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731612913197 2024-11-14T19:35:14,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:14,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:14,516 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:14,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:14,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:14,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:14,516 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:14,519 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:14,519 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/B is initiating minor compaction (all files) 2024-11-14T19:35:14,519 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/B in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,519 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3f38357279f347038ae0752162c66a76, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d2ab8cabfcfc44c39cf63d4aafa45a92, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d3d9af0d31434734b9e73f1574a9f09a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=34.5 K 2024-11-14T19:35:14,520 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f38357279f347038ae0752162c66a76, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731612912079 2024-11-14T19:35:14,520 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting d2ab8cabfcfc44c39cf63d4aafa45a92, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731612912428 2024-11-14T19:35:14,521 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d9af0d31434734b9e73f1574a9f09a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731612913197 2024-11-14T19:35:14,529 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:14,529 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/467d853e2a904fce8696f623e0f893f2 is 50, key is test_row_0/A:col10/1731612913533/Put/seqid=0 2024-11-14T19:35:14,545 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:14,546 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/54b7a18054644960b59003e77bd28404 is 50, key is test_row_0/B:col10/1731612913533/Put/seqid=0 2024-11-14T19:35:14,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-14T19:35:14,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,548 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T19:35:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:14,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/0e8137cb148441f7bce3f7b764b5f51f is 50, key is test_row_0/A:col10/1731612913538/Put/seqid=0 2024-11-14T19:35:14,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741927_1103 (size=13323) 2024-11-14T19:35:14,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741929_1105 (size=12301) 2024-11-14T19:35:14,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741928_1104 (size=13323) 2024-11-14T19:35:14,578 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/467d853e2a904fce8696f623e0f893f2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/467d853e2a904fce8696f623e0f893f2 2024-11-14T19:35:14,589 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/A of 21b827b5bf7b423416191650584b91ae into 467d853e2a904fce8696f623e0f893f2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:14,589 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:14,589 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/A, priority=13, startTime=1731612914511; duration=0sec 2024-11-14T19:35:14,589 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:14,589 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:14,589 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:14,591 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:14,592 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 21b827b5bf7b423416191650584b91ae/C is initiating minor compaction (all files) 2024-11-14T19:35:14,592 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 21b827b5bf7b423416191650584b91ae/C in TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:14,592 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/28457c92767b45899b103a65425c19be, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/7a67244f3a7c4027bbbcc5d776c56a4c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/69b9f779d773445ca22a3f19b0e50bad] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp, totalSize=34.5 K 2024-11-14T19:35:14,592 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28457c92767b45899b103a65425c19be, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731612912079 2024-11-14T19:35:14,594 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a67244f3a7c4027bbbcc5d776c56a4c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1731612912428 2024-11-14T19:35:14,594 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69b9f779d773445ca22a3f19b0e50bad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731612913197 2024-11-14T19:35:14,609 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 21b827b5bf7b423416191650584b91ae#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:14,609 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/15252aec0113483d80a04b1f8f9114a9 is 50, key is test_row_0/C:col10/1731612913533/Put/seqid=0 2024-11-14T19:35:14,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741930_1106 (size=13323) 2024-11-14T19:35:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:14,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. as already flushing 2024-11-14T19:35:14,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T19:35:14,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612914755, max timestamp=1731612914782Num uniques 859; 2024-11-14T19:35:14,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612914755, max timestamp=1731612914782Num uniques 859; 2024-11-14T19:35:14,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612914755, max timestamp=1731612914782Num uniques 859; 2024-11-14T19:35:14,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:35:14,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:35:14,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:35:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731612914827, max timestamp=1731612914860Num uniques 1018; 2024-11-14T19:35:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T19:35:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731612914827, max timestamp=1731612914860Num uniques 1018; 2024-11-14T19:35:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731612914827, max timestamp=1731612914860Num uniques 1018; 2024-11-14T19:35:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T19:35:14,864 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T19:35:14,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56414 deadline: 1731612974945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:14,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731612914924, max timestamp=1731612914944Num uniques 1077; 2024-11-14T19:35:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:14,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731612914924, max timestamp=1731612914944Num uniques 1077; 2024-11-14T19:35:14,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731612914924, max timestamp=1731612914944Num uniques 1077; 2024-11-14T19:35:14,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T19:35:14,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T19:35:14,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-14T19:35:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:14,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:14,970 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/0e8137cb148441f7bce3f7b764b5f51f 2024-11-14T19:35:14,977 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/54b7a18054644960b59003e77bd28404 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/54b7a18054644960b59003e77bd28404 2024-11-14T19:35:14,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/c130d4987ae843ca8b0ee3bf3fb9d75b is 50, key is test_row_0/B:col10/1731612913538/Put/seqid=0 2024-11-14T19:35:14,985 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/B of 21b827b5bf7b423416191650584b91ae into 54b7a18054644960b59003e77bd28404(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:14,985 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:14,985 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/B, priority=13, startTime=1731612914516; duration=0sec 2024-11-14T19:35:14,986 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:14,986 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:14,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741931_1107 (size=12301) 2024-11-14T19:35:14,990 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/c130d4987ae843ca8b0ee3bf3fb9d75b 2024-11-14T19:35:15,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/ce2ba4533d4b4fdeaeff9e1bca509b13 is 50, key is test_row_0/C:col10/1731612913538/Put/seqid=0 2024-11-14T19:35:15,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741932_1108 (size=12301) 2024-11-14T19:35:15,032 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/15252aec0113483d80a04b1f8f9114a9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/15252aec0113483d80a04b1f8f9114a9 2024-11-14T19:35:15,043 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 21b827b5bf7b423416191650584b91ae/C of 21b827b5bf7b423416191650584b91ae into 15252aec0113483d80a04b1f8f9114a9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:15,043 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:15,043 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., storeName=21b827b5bf7b423416191650584b91ae/C, priority=13, startTime=1731612914516; duration=0sec 2024-11-14T19:35:15,044 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:15,044 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612915125, max timestamp=1731612915159Num uniques 1086; 2024-11-14T19:35:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T19:35:15,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612915125, max timestamp=1731612915159Num uniques 1086; 2024-11-14T19:35:15,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T19:35:15,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612915125, max timestamp=1731612915159Num uniques 1086; 2024-11-14T19:35:15,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-14T19:35:15,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731612915213, max timestamp=1731612915243Num uniques 1045; 2024-11-14T19:35:15,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731612915213, max timestamp=1731612915243Num uniques 1045; 2024-11-14T19:35:15,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-14T19:35:15,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-14T19:35:15,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731612915213, max timestamp=1731612915243Num uniques 1045; 2024-11-14T19:35:15,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T19:35:15,302 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-14T19:35:15,302 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,303 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-14T19:35:15,303 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,306 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-14T19:35:15,306 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,306 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-14T19:35:15,306 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,307 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-14T19:35:15,307 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,307 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,307 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,307 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,307 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,307 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,308 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,308 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,308 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,308 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,309 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,355 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-14T19:35:15,355 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,355 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,355 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,416 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/ce2ba4533d4b4fdeaeff9e1bca509b13 2024-11-14T19:35:15,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/0e8137cb148441f7bce3f7b764b5f51f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/0e8137cb148441f7bce3f7b764b5f51f 2024-11-14T19:35:15,427 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/0e8137cb148441f7bce3f7b764b5f51f, entries=150, sequenceid=592, filesize=12.0 K 2024-11-14T19:35:15,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/c130d4987ae843ca8b0ee3bf3fb9d75b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c130d4987ae843ca8b0ee3bf3fb9d75b 2024-11-14T19:35:15,434 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c130d4987ae843ca8b0ee3bf3fb9d75b, entries=150, sequenceid=592, filesize=12.0 K 2024-11-14T19:35:15,435 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-14T19:35:15,435 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,435 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,435 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,435 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/ce2ba4533d4b4fdeaeff9e1bca509b13 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ce2ba4533d4b4fdeaeff9e1bca509b13 2024-11-14T19:35:15,442 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ce2ba4533d4b4fdeaeff9e1bca509b13, entries=150, sequenceid=592, filesize=12.0 K 2024-11-14T19:35:15,443 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=118.53 KB/121370 for 21b827b5bf7b423416191650584b91ae in 895ms, sequenceid=592, compaction requested=false 2024-11-14T19:35:15,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:15,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:15,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-14T19:35:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-14T19:35:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:15,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=125.23 KB heapSize=351.92 KB 2024-11-14T19:35:15,445 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-14T19:35:15,445 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,445 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:15,445 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:15,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:15,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-14T19:35:15,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-11-14T19:35:15,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8240 sec 2024-11-14T19:35:15,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/8b8e1ef5fd2348559d27f52db77b4a6a is 50, key is test_row_0/A:col10/1731612915443/Put/seqid=0 2024-11-14T19:35:15,455 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-14T19:35:15,455 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:15,456 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,456 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741933_1109 (size=12301) 2024-11-14T19:35:15,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-14T19:35:15,774 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:15,774 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T19:35:15,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-14T19:35:15,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 143 2024-11-14T19:35:15,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 235 2024-11-14T19:35:15,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-14T19:35:15,775 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-14T19:35:15,776 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T19:35:15,776 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2110 2024-11-14T19:35:15,776 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2046 2024-11-14T19:35:15,776 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T19:35:15,776 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 947 2024-11-14T19:35:15,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2840 rows 2024-11-14T19:35:15,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 938 2024-11-14T19:35:15,777 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2813 rows 2024-11-14T19:35:15,777 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:35:15,778 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:35:15,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:15,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:15,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T19:35:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T19:35:15,791 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612915791"}]},"ts":"1731612915791"} 2024-11-14T19:35:15,793 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T19:35:15,793 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T19:35:15,795 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:35:15,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, UNASSIGN}] 2024-11-14T19:35:15,801 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, UNASSIGN 2024-11-14T19:35:15,802 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=21b827b5bf7b423416191650584b91ae, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:15,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, UNASSIGN because future has completed 2024-11-14T19:35:15,806 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:35:15,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:35:15,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/8b8e1ef5fd2348559d27f52db77b4a6a 2024-11-14T19:35:15,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/aa78a91b1e944905b3ecc2077d25b997 is 50, key is test_row_0/B:col10/1731612915443/Put/seqid=0 2024-11-14T19:35:15,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741934_1110 (size=12301) 2024-11-14T19:35:15,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T19:35:15,965 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:15,965 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:35:15,966 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 21b827b5bf7b423416191650584b91ae, disabling compactions & flushes 2024-11-14T19:35:15,966 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:16,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T19:35:16,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/aa78a91b1e944905b3ecc2077d25b997 2024-11-14T19:35:16,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/4cc0d9dc690f48ce9319cdffa9b0a382 is 50, key is test_row_0/C:col10/1731612915443/Put/seqid=0 2024-11-14T19:35:16,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741935_1111 (size=12301) 2024-11-14T19:35:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T19:35:16,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=698 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/4cc0d9dc690f48ce9319cdffa9b0a382 2024-11-14T19:35:16,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/8b8e1ef5fd2348559d27f52db77b4a6a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8b8e1ef5fd2348559d27f52db77b4a6a 2024-11-14T19:35:16,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8b8e1ef5fd2348559d27f52db77b4a6a, entries=150, sequenceid=698, filesize=12.0 K 2024-11-14T19:35:16,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/aa78a91b1e944905b3ecc2077d25b997 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/aa78a91b1e944905b3ecc2077d25b997 2024-11-14T19:35:16,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/aa78a91b1e944905b3ecc2077d25b997, entries=150, sequenceid=698, filesize=12.0 K 2024-11-14T19:35:16,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/4cc0d9dc690f48ce9319cdffa9b0a382 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4cc0d9dc690f48ce9319cdffa9b0a382 2024-11-14T19:35:16,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4cc0d9dc690f48ce9319cdffa9b0a382, entries=150, sequenceid=698, filesize=12.0 K 2024-11-14T19:35:16,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~351.88 KB/360320, currentSize=6.71 KB/6870 for 21b827b5bf7b423416191650584b91ae in 1318ms, sequenceid=698, compaction requested=true 2024-11-14T19:35:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 21b827b5bf7b423416191650584b91ae: 2024-11-14T19:35:16,762 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:16,762 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:16,762 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. after waiting 0 ms 2024-11-14T19:35:16,762 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:16,762 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. because compaction request was cancelled 2024-11-14T19:35:16,762 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 21b827b5bf7b423416191650584b91ae 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-14T19:35:16,762 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:A 2024-11-14T19:35:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:A, priority=-2147483648, current under compaction store size is 0 2024-11-14T19:35:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:16,763 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=A 2024-11-14T19:35:16,763 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:16,763 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=B 2024-11-14T19:35:16,763 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:16,763 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 21b827b5bf7b423416191650584b91ae, store=C 2024-11-14T19:35:16,763 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:16,764 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. because compaction request was cancelled 2024-11-14T19:35:16,764 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:B 2024-11-14T19:35:16,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:B, priority=-2147483648, current under compaction store size is 0 2024-11-14T19:35:16,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:16,764 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. because compaction request was cancelled 2024-11-14T19:35:16,764 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 21b827b5bf7b423416191650584b91ae:C 2024-11-14T19:35:16,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 21b827b5bf7b423416191650584b91ae:C, priority=-2147483648, current under compaction store size is 0 2024-11-14T19:35:16,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:16,768 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/be512a3444f04e04b79afe8c01d73b39 is 50, key is test_row_2/A:col10/1731612915454/Put/seqid=0 2024-11-14T19:35:16,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741936_1112 (size=7415) 2024-11-14T19:35:16,785 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/be512a3444f04e04b79afe8c01d73b39 2024-11-14T19:35:16,803 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/0819fae2b41247ebb488214dd2c62123 is 50, key is test_row_2/B:col10/1731612915454/Put/seqid=0 2024-11-14T19:35:16,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741937_1113 (size=7415) 2024-11-14T19:35:16,827 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/0819fae2b41247ebb488214dd2c62123 2024-11-14T19:35:16,849 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/a3f866b57ddb48f793b1ef190364955c is 50, key is test_row_2/C:col10/1731612915454/Put/seqid=0 2024-11-14T19:35:16,860 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T19:35:16,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741938_1114 (size=7415) 2024-11-14T19:35:16,880 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/a3f866b57ddb48f793b1ef190364955c 2024-11-14T19:35:16,898 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/A/be512a3444f04e04b79afe8c01d73b39 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/be512a3444f04e04b79afe8c01d73b39 2024-11-14T19:35:16,911 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/be512a3444f04e04b79afe8c01d73b39, entries=50, sequenceid=702, filesize=7.2 K 2024-11-14T19:35:16,915 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/B/0819fae2b41247ebb488214dd2c62123 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/0819fae2b41247ebb488214dd2c62123 2024-11-14T19:35:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T19:35:16,933 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/0819fae2b41247ebb488214dd2c62123, entries=50, sequenceid=702, filesize=7.2 K 2024-11-14T19:35:16,934 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/.tmp/C/a3f866b57ddb48f793b1ef190364955c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/a3f866b57ddb48f793b1ef190364955c 2024-11-14T19:35:16,945 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/a3f866b57ddb48f793b1ef190364955c, entries=50, sequenceid=702, filesize=7.2 K 2024-11-14T19:35:16,947 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 21b827b5bf7b423416191650584b91ae in 184ms, sequenceid=702, compaction requested=true 2024-11-14T19:35:16,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/ccceba6b5e6e4505829f3ebcfd7c535a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/309bcb6fa2c74ba29421f854821f39d7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/50d50cf61fae4bc089d8380262c88b72, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1c41ef8790d64552bbcae76f96e2cb68, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b4f1e62948774839a3d0b99ff1b88f7a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/774d2424330646f1bc0edb04e7895be9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/89ab5e127be24b5ebdee1beaccda3c4d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2308d1f5a42b404ab28ca9edd5065dc2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b5613746c97642ec8836450609f63d7b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/151081dfc2364cc3813199be18104d4e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2070d735dedd41458f032c7434ffbf27, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a82becb0e064716bfdf90da74f51a95, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c562ab6425ef4ebebff77c2508cf5ba8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fc3a68665e3544e8b5f66e1de50556c2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c94ea7fe9d7d410dbe55559ddf67e183, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/22768114419a4c3d9c5075a74bc8ee22, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fe86626ee3ec4f1dbb6b68d6f7eba124, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/bfdc516536c8471da10b60ed1bd61b08, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/203fd011f2884ecca582e4d9f18ecf06, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8ae8b0c3b510438f9b12cfba9777fedd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/99bfd5abb51343eb835d041f3d0e1e0f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/68f4037872f7419a9ec9e0d27e1056b3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1fb814cc2fa1415d9487430cc468ff6f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e38e37de00d34813b0c5da2528a4d930, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/9bc9b51fa56149638a5875af095e8efc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d16d090797064869b75887860ba44556, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d195f1aa5859464f85587ec9e7d4dfb7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e52adad4f747439f88d89ef35a7f980d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/a5e4406a2c0b43f8b4f6a7b5cd26c941, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a6107b094ed494287396537dd5d51a1] to archive 2024-11-14T19:35:16,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:35:16,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/ccceba6b5e6e4505829f3ebcfd7c535a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/ccceba6b5e6e4505829f3ebcfd7c535a 2024-11-14T19:35:16,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/309bcb6fa2c74ba29421f854821f39d7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/309bcb6fa2c74ba29421f854821f39d7 2024-11-14T19:35:16,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/50d50cf61fae4bc089d8380262c88b72 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/50d50cf61fae4bc089d8380262c88b72 2024-11-14T19:35:16,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1c41ef8790d64552bbcae76f96e2cb68 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1c41ef8790d64552bbcae76f96e2cb68 2024-11-14T19:35:16,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b4f1e62948774839a3d0b99ff1b88f7a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b4f1e62948774839a3d0b99ff1b88f7a 2024-11-14T19:35:17,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/774d2424330646f1bc0edb04e7895be9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/774d2424330646f1bc0edb04e7895be9 2024-11-14T19:35:17,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/89ab5e127be24b5ebdee1beaccda3c4d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/89ab5e127be24b5ebdee1beaccda3c4d 2024-11-14T19:35:17,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2308d1f5a42b404ab28ca9edd5065dc2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2308d1f5a42b404ab28ca9edd5065dc2 2024-11-14T19:35:17,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b5613746c97642ec8836450609f63d7b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/b5613746c97642ec8836450609f63d7b 2024-11-14T19:35:17,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/151081dfc2364cc3813199be18104d4e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/151081dfc2364cc3813199be18104d4e 2024-11-14T19:35:17,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2070d735dedd41458f032c7434ffbf27 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/2070d735dedd41458f032c7434ffbf27 2024-11-14T19:35:17,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a82becb0e064716bfdf90da74f51a95 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a82becb0e064716bfdf90da74f51a95 2024-11-14T19:35:17,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c562ab6425ef4ebebff77c2508cf5ba8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c562ab6425ef4ebebff77c2508cf5ba8 2024-11-14T19:35:17,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fc3a68665e3544e8b5f66e1de50556c2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fc3a68665e3544e8b5f66e1de50556c2 2024-11-14T19:35:17,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c94ea7fe9d7d410dbe55559ddf67e183 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/c94ea7fe9d7d410dbe55559ddf67e183 2024-11-14T19:35:17,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/22768114419a4c3d9c5075a74bc8ee22 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/22768114419a4c3d9c5075a74bc8ee22 2024-11-14T19:35:17,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fe86626ee3ec4f1dbb6b68d6f7eba124 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/fe86626ee3ec4f1dbb6b68d6f7eba124 2024-11-14T19:35:17,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/bfdc516536c8471da10b60ed1bd61b08 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/bfdc516536c8471da10b60ed1bd61b08 2024-11-14T19:35:17,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/203fd011f2884ecca582e4d9f18ecf06 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/203fd011f2884ecca582e4d9f18ecf06 2024-11-14T19:35:17,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8ae8b0c3b510438f9b12cfba9777fedd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8ae8b0c3b510438f9b12cfba9777fedd 2024-11-14T19:35:17,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/99bfd5abb51343eb835d041f3d0e1e0f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/99bfd5abb51343eb835d041f3d0e1e0f 2024-11-14T19:35:17,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/68f4037872f7419a9ec9e0d27e1056b3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/68f4037872f7419a9ec9e0d27e1056b3 2024-11-14T19:35:17,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1fb814cc2fa1415d9487430cc468ff6f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/1fb814cc2fa1415d9487430cc468ff6f 2024-11-14T19:35:17,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e38e37de00d34813b0c5da2528a4d930 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e38e37de00d34813b0c5da2528a4d930 2024-11-14T19:35:17,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/9bc9b51fa56149638a5875af095e8efc to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/9bc9b51fa56149638a5875af095e8efc 2024-11-14T19:35:17,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d16d090797064869b75887860ba44556 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d16d090797064869b75887860ba44556 2024-11-14T19:35:17,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d195f1aa5859464f85587ec9e7d4dfb7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/d195f1aa5859464f85587ec9e7d4dfb7 2024-11-14T19:35:17,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e52adad4f747439f88d89ef35a7f980d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/e52adad4f747439f88d89ef35a7f980d 2024-11-14T19:35:17,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/a5e4406a2c0b43f8b4f6a7b5cd26c941 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/a5e4406a2c0b43f8b4f6a7b5cd26c941 2024-11-14T19:35:17,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a6107b094ed494287396537dd5d51a1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/7a6107b094ed494287396537dd5d51a1 2024-11-14T19:35:17,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7ccc427375545a5b42537db21526ca4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/104d0baed3da43529511e535795c0b72, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9bff8ad8a05c410ab6a25c5bc7baa920, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/74aebc3d657a46978290085cd7a4967d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c1578d7d361c4d769ecd7ddf129b6c1a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/8d83012693c049f68dc16af714f427c5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/46f41dcbfb254682823aa38744dde1be, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/016721c5794041c18824db5e36a1dc30, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/77f07349278844e58f606cb965c13942, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/25f30c76610543d89aa94d5ad121c9af, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6061fda91b8d4dda94aaefaf7a7a16ae, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dca593af4fe9448f86190cae77925b57, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dd19da9a84874a4f858f92c097a7ab4e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6e932301f9c642d18884197f3e1a5719, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/cf0dac03577743a9a8a7de678c8d07bb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b8bc1eb8977145438ae17514a9bc9acb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9b64acfb9d8445b68fa5e887fcc1d3e2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/ee3e45551e8d4cabbb089e95a18d4336, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/627492b26ef246d584d02eb0e9d32510, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/980436ee69b94518ace5f1ae5ff63c48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7883d937eb74a9aa419d145dd7c71b1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b9b3f4eeb12e48a3a6828617a90b9032, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/fc86e66065124d22acf3c97f7df8aed6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d5129f9d9f664905b2158b526ffdf5dd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9ebad643f218471bb9ba68a7581c441d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/f4e2a872b91f4ebb81fadbbd90849a0c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3f38357279f347038ae0752162c66a76, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3a897ca4ef734f2fbd2aaf917486ad1e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d2ab8cabfcfc44c39cf63d4aafa45a92, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d3d9af0d31434734b9e73f1574a9f09a] to archive 2024-11-14T19:35:17,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:35:17,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7ccc427375545a5b42537db21526ca4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7ccc427375545a5b42537db21526ca4 2024-11-14T19:35:17,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/104d0baed3da43529511e535795c0b72 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/104d0baed3da43529511e535795c0b72 2024-11-14T19:35:17,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9bff8ad8a05c410ab6a25c5bc7baa920 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9bff8ad8a05c410ab6a25c5bc7baa920 2024-11-14T19:35:17,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/74aebc3d657a46978290085cd7a4967d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/74aebc3d657a46978290085cd7a4967d 2024-11-14T19:35:17,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c1578d7d361c4d769ecd7ddf129b6c1a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c1578d7d361c4d769ecd7ddf129b6c1a 2024-11-14T19:35:17,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/8d83012693c049f68dc16af714f427c5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/8d83012693c049f68dc16af714f427c5 2024-11-14T19:35:17,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/46f41dcbfb254682823aa38744dde1be to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/46f41dcbfb254682823aa38744dde1be 2024-11-14T19:35:17,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/016721c5794041c18824db5e36a1dc30 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/016721c5794041c18824db5e36a1dc30 2024-11-14T19:35:17,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/77f07349278844e58f606cb965c13942 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/77f07349278844e58f606cb965c13942 2024-11-14T19:35:17,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/25f30c76610543d89aa94d5ad121c9af to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/25f30c76610543d89aa94d5ad121c9af 2024-11-14T19:35:17,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6061fda91b8d4dda94aaefaf7a7a16ae to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6061fda91b8d4dda94aaefaf7a7a16ae 2024-11-14T19:35:17,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dca593af4fe9448f86190cae77925b57 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dca593af4fe9448f86190cae77925b57 2024-11-14T19:35:17,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dd19da9a84874a4f858f92c097a7ab4e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/dd19da9a84874a4f858f92c097a7ab4e 2024-11-14T19:35:17,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6e932301f9c642d18884197f3e1a5719 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/6e932301f9c642d18884197f3e1a5719 2024-11-14T19:35:17,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/cf0dac03577743a9a8a7de678c8d07bb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/cf0dac03577743a9a8a7de678c8d07bb 2024-11-14T19:35:17,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b8bc1eb8977145438ae17514a9bc9acb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b8bc1eb8977145438ae17514a9bc9acb 2024-11-14T19:35:17,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9b64acfb9d8445b68fa5e887fcc1d3e2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9b64acfb9d8445b68fa5e887fcc1d3e2 2024-11-14T19:35:17,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/ee3e45551e8d4cabbb089e95a18d4336 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/ee3e45551e8d4cabbb089e95a18d4336 2024-11-14T19:35:17,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/627492b26ef246d584d02eb0e9d32510 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/627492b26ef246d584d02eb0e9d32510 2024-11-14T19:35:17,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/980436ee69b94518ace5f1ae5ff63c48 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/980436ee69b94518ace5f1ae5ff63c48 2024-11-14T19:35:17,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7883d937eb74a9aa419d145dd7c71b1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/a7883d937eb74a9aa419d145dd7c71b1 2024-11-14T19:35:17,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b9b3f4eeb12e48a3a6828617a90b9032 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/b9b3f4eeb12e48a3a6828617a90b9032 2024-11-14T19:35:17,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/fc86e66065124d22acf3c97f7df8aed6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/fc86e66065124d22acf3c97f7df8aed6 2024-11-14T19:35:17,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d5129f9d9f664905b2158b526ffdf5dd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d5129f9d9f664905b2158b526ffdf5dd 2024-11-14T19:35:17,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9ebad643f218471bb9ba68a7581c441d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/9ebad643f218471bb9ba68a7581c441d 2024-11-14T19:35:17,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/f4e2a872b91f4ebb81fadbbd90849a0c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/f4e2a872b91f4ebb81fadbbd90849a0c 2024-11-14T19:35:17,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3f38357279f347038ae0752162c66a76 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3f38357279f347038ae0752162c66a76 2024-11-14T19:35:17,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3a897ca4ef734f2fbd2aaf917486ad1e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/3a897ca4ef734f2fbd2aaf917486ad1e 2024-11-14T19:35:17,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d2ab8cabfcfc44c39cf63d4aafa45a92 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d2ab8cabfcfc44c39cf63d4aafa45a92 2024-11-14T19:35:17,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d3d9af0d31434734b9e73f1574a9f09a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/d3d9af0d31434734b9e73f1574a9f09a 2024-11-14T19:35:17,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/f1dafd5e957648acaa21121d21289d9e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/770faf9bf313448f960dbd4cee815824, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3cd427ce120a4aebacdd836508277aec, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e6d24db87a58482baa5c11230ee1b5dd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/464d85af0e7e4247abd03f0810822764, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e2a7fdb41cd44ce8936b06f3ec449a20, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1c23ac1e24b54481a35ad03c45ce54d0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1cad4c79100443c7a644445cc8e00be3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e1ec43a604bc473ab0b6d6845f018880, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3d69188c35614e3595eb6a4aa91f8c97, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/6c0f70566b2f40cf8952dd662fb423df, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/cf3f96ae9dfd4c6783ce204246a4966e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/265e04e8a28444df872a4f24f0a91aaa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c2001cb3775541158b02fad005687e9d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e47434cd4bc84097b2ec0655e89bc88b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8d563d3aa0a46f2b94d1d032f8e662f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c0838421e38a4467984b4196abbe316e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e00d47fdf7d344acacd9db972506f8fd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/18c645967e7e40718ade6f3e11d8069d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/60612e424ece42c3bb46243f5deadf42, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/910262081977409abe5a92670abe376d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4de7c1272325489298f986684955b400, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8b39ad714eb433294ad86c3a0fd9217, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/b8ed4fa41360457ca3f4d764cbdaf3a9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/dd015a437069433e9d3a1a185166723b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ad864453eb924de39835d6e2072c26cd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/28457c92767b45899b103a65425c19be, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/77b009860a2e4ad5a91f3059fb3c3645, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/7a67244f3a7c4027bbbcc5d776c56a4c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/69b9f779d773445ca22a3f19b0e50bad] to archive 2024-11-14T19:35:17,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:35:17,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/f1dafd5e957648acaa21121d21289d9e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/f1dafd5e957648acaa21121d21289d9e 2024-11-14T19:35:17,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/770faf9bf313448f960dbd4cee815824 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/770faf9bf313448f960dbd4cee815824 2024-11-14T19:35:17,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3cd427ce120a4aebacdd836508277aec to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3cd427ce120a4aebacdd836508277aec 2024-11-14T19:35:17,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e6d24db87a58482baa5c11230ee1b5dd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e6d24db87a58482baa5c11230ee1b5dd 2024-11-14T19:35:17,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/464d85af0e7e4247abd03f0810822764 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/464d85af0e7e4247abd03f0810822764 2024-11-14T19:35:17,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e2a7fdb41cd44ce8936b06f3ec449a20 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e2a7fdb41cd44ce8936b06f3ec449a20 2024-11-14T19:35:17,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1c23ac1e24b54481a35ad03c45ce54d0 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1c23ac1e24b54481a35ad03c45ce54d0 2024-11-14T19:35:17,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1cad4c79100443c7a644445cc8e00be3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/1cad4c79100443c7a644445cc8e00be3 2024-11-14T19:35:17,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e1ec43a604bc473ab0b6d6845f018880 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e1ec43a604bc473ab0b6d6845f018880 2024-11-14T19:35:17,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3d69188c35614e3595eb6a4aa91f8c97 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/3d69188c35614e3595eb6a4aa91f8c97 2024-11-14T19:35:17,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/6c0f70566b2f40cf8952dd662fb423df to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/6c0f70566b2f40cf8952dd662fb423df 2024-11-14T19:35:17,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/cf3f96ae9dfd4c6783ce204246a4966e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/cf3f96ae9dfd4c6783ce204246a4966e 2024-11-14T19:35:17,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/265e04e8a28444df872a4f24f0a91aaa to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/265e04e8a28444df872a4f24f0a91aaa 2024-11-14T19:35:17,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c2001cb3775541158b02fad005687e9d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c2001cb3775541158b02fad005687e9d 2024-11-14T19:35:17,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e47434cd4bc84097b2ec0655e89bc88b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e47434cd4bc84097b2ec0655e89bc88b 2024-11-14T19:35:17,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8d563d3aa0a46f2b94d1d032f8e662f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8d563d3aa0a46f2b94d1d032f8e662f 2024-11-14T19:35:17,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c0838421e38a4467984b4196abbe316e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/c0838421e38a4467984b4196abbe316e 2024-11-14T19:35:17,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e00d47fdf7d344acacd9db972506f8fd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/e00d47fdf7d344acacd9db972506f8fd 2024-11-14T19:35:17,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/18c645967e7e40718ade6f3e11d8069d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/18c645967e7e40718ade6f3e11d8069d 2024-11-14T19:35:17,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/60612e424ece42c3bb46243f5deadf42 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/60612e424ece42c3bb46243f5deadf42 2024-11-14T19:35:17,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/910262081977409abe5a92670abe376d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/910262081977409abe5a92670abe376d 2024-11-14T19:35:17,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4de7c1272325489298f986684955b400 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4de7c1272325489298f986684955b400 2024-11-14T19:35:17,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8b39ad714eb433294ad86c3a0fd9217 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/d8b39ad714eb433294ad86c3a0fd9217 2024-11-14T19:35:17,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/b8ed4fa41360457ca3f4d764cbdaf3a9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/b8ed4fa41360457ca3f4d764cbdaf3a9 2024-11-14T19:35:17,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/dd015a437069433e9d3a1a185166723b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/dd015a437069433e9d3a1a185166723b 2024-11-14T19:35:17,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ad864453eb924de39835d6e2072c26cd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ad864453eb924de39835d6e2072c26cd 2024-11-14T19:35:17,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/28457c92767b45899b103a65425c19be to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/28457c92767b45899b103a65425c19be 2024-11-14T19:35:17,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/77b009860a2e4ad5a91f3059fb3c3645 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/77b009860a2e4ad5a91f3059fb3c3645 2024-11-14T19:35:17,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/7a67244f3a7c4027bbbcc5d776c56a4c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/7a67244f3a7c4027bbbcc5d776c56a4c 2024-11-14T19:35:17,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/69b9f779d773445ca22a3f19b0e50bad to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/69b9f779d773445ca22a3f19b0e50bad 2024-11-14T19:35:17,327 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/recovered.edits/705.seqid, newMaxSeqId=705, maxSeqId=1 2024-11-14T19:35:17,329 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. 2024-11-14T19:35:17,329 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 21b827b5bf7b423416191650584b91ae: Waiting for close lock at 1731612915966Running coprocessor pre-close hooks at 1731612915966Disabling compacts and flushes for region at 1731612915966Disabling writes for close at 1731612916762 (+796 ms)Obtaining lock to block concurrent updates at 1731612916762Preparing flush snapshotting stores in 21b827b5bf7b423416191650584b91ae at 1731612916762Finished memstore snapshotting TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1731612916763 (+1 ms)Flushing stores of TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae. at 1731612916764 (+1 ms)Flushing 21b827b5bf7b423416191650584b91ae/A: creating writer at 1731612916764Flushing 21b827b5bf7b423416191650584b91ae/A: appending metadata at 1731612916767 (+3 ms)Flushing 21b827b5bf7b423416191650584b91ae/A: closing flushed file at 1731612916767Flushing 21b827b5bf7b423416191650584b91ae/B: creating writer at 1731612916797 (+30 ms)Flushing 21b827b5bf7b423416191650584b91ae/B: appending metadata at 1731612916802 (+5 ms)Flushing 21b827b5bf7b423416191650584b91ae/B: closing flushed file at 1731612916802Flushing 21b827b5bf7b423416191650584b91ae/C: creating writer at 1731612916835 (+33 ms)Flushing 21b827b5bf7b423416191650584b91ae/C: appending metadata at 1731612916848 (+13 ms)Flushing 21b827b5bf7b423416191650584b91ae/C: closing flushed file at 1731612916848Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22bd9d2f: reopening flushed file at 1731612916894 (+46 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@496cf029: reopening flushed file at 1731612916912 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56338847: reopening flushed file at 1731612916933 (+21 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 21b827b5bf7b423416191650584b91ae in 184ms, sequenceid=702, compaction requested=true at 1731612916947 (+14 ms)Writing region close event to WAL at 1731612917313 (+366 ms)Running coprocessor post-close hooks at 1731612917329 (+16 ms)Closed at 1731612917329 2024-11-14T19:35:17,341 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:17,342 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=21b827b5bf7b423416191650584b91ae, regionState=CLOSED 2024-11-14T19:35:17,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:35:17,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-14T19:35:17,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 21b827b5bf7b423416191650584b91ae, server=867b237d0fa7,38601,1731612889637 in 1.5500 sec 2024-11-14T19:35:17,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-14T19:35:17,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=21b827b5bf7b423416191650584b91ae, UNASSIGN in 1.5590 sec 2024-11-14T19:35:17,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-14T19:35:17,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5780 sec 2024-11-14T19:35:17,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612917378"}]},"ts":"1731612917378"} 2024-11-14T19:35:17,381 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T19:35:17,381 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T19:35:17,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5980 sec 2024-11-14T19:35:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-14T19:35:17,936 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:17,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T19:35:17,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:17,950 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T19:35:17,954 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:17,980 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:17,999 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/recovered.edits] 2024-11-14T19:35:18,005 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/0e8137cb148441f7bce3f7b764b5f51f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/0e8137cb148441f7bce3f7b764b5f51f 2024-11-14T19:35:18,007 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/467d853e2a904fce8696f623e0f893f2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/467d853e2a904fce8696f623e0f893f2 2024-11-14T19:35:18,010 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8b8e1ef5fd2348559d27f52db77b4a6a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/8b8e1ef5fd2348559d27f52db77b4a6a 2024-11-14T19:35:18,013 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/be512a3444f04e04b79afe8c01d73b39 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/A/be512a3444f04e04b79afe8c01d73b39 2024-11-14T19:35:18,019 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/0819fae2b41247ebb488214dd2c62123 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/0819fae2b41247ebb488214dd2c62123 2024-11-14T19:35:18,022 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/54b7a18054644960b59003e77bd28404 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/54b7a18054644960b59003e77bd28404 2024-11-14T19:35:18,025 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/aa78a91b1e944905b3ecc2077d25b997 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/aa78a91b1e944905b3ecc2077d25b997 2024-11-14T19:35:18,027 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c130d4987ae843ca8b0ee3bf3fb9d75b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/B/c130d4987ae843ca8b0ee3bf3fb9d75b 2024-11-14T19:35:18,031 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/15252aec0113483d80a04b1f8f9114a9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/15252aec0113483d80a04b1f8f9114a9 2024-11-14T19:35:18,036 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4cc0d9dc690f48ce9319cdffa9b0a382 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/4cc0d9dc690f48ce9319cdffa9b0a382 2024-11-14T19:35:18,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/a3f866b57ddb48f793b1ef190364955c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/a3f866b57ddb48f793b1ef190364955c 2024-11-14T19:35:18,040 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ce2ba4533d4b4fdeaeff9e1bca509b13 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/C/ce2ba4533d4b4fdeaeff9e1bca509b13 2024-11-14T19:35:18,049 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/recovered.edits/705.seqid to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae/recovered.edits/705.seqid 2024-11-14T19:35:18,050 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/21b827b5bf7b423416191650584b91ae 2024-11-14T19:35:18,050 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T19:35:18,058 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T19:35:18,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-14T19:35:18,070 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T19:35:18,115 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T19:35:18,117 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:18,117 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T19:35:18,117 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731612918117"}]},"ts":"9223372036854775807"} 2024-11-14T19:35:18,125 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T19:35:18,125 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 21b827b5bf7b423416191650584b91ae, NAME => 'TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T19:35:18,126 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T19:35:18,127 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731612918126"}]},"ts":"9223372036854775807"} 2024-11-14T19:35:18,130 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T19:35:18,132 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:18,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 191 msec 2024-11-14T19:35:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-14T19:35:18,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T19:35:18,275 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:18,292 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1421657582_22 at /127.0.0.1:38618 [Waiting for operation #310] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;867b237d0fa7:38601-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1742159255_22 at /127.0.0.1:42840 [Waiting for operation #328] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 298) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5907 (was 6524) 2024-11-14T19:35:18,304 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=5906 2024-11-14T19:35:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:35:18,310 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:35:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:18,313 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T19:35:18,314 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:18,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-14T19:35:18,315 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T19:35:18,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T19:35:18,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741939_1115 (size=1061) 2024-11-14T19:35:18,351 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:35:18,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741940_1116 (size=53) 2024-11-14T19:35:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T19:35:18,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T19:35:18,772 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:35:18,772 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 965abd24f852442dbb51955455ef7db5, disabling compactions & flushes 2024-11-14T19:35:18,772 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:18,773 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:18,773 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. after waiting 0 ms 2024-11-14T19:35:18,773 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:18,773 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:18,773 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 965abd24f852442dbb51955455ef7db5: Waiting for close lock at 1731612918772Disabling compacts and flushes for region at 1731612918772Disabling writes for close at 1731612918773 (+1 ms)Writing region close event to WAL at 1731612918773Closed at 1731612918773 2024-11-14T19:35:18,775 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T19:35:18,775 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731612918775"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731612918775"}]},"ts":"1731612918775"} 2024-11-14T19:35:18,780 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T19:35:18,782 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T19:35:18,782 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612918782"}]},"ts":"1731612918782"} 2024-11-14T19:35:18,785 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T19:35:18,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, ASSIGN}] 2024-11-14T19:35:18,787 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, ASSIGN 2024-11-14T19:35:18,788 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, ASSIGN; state=OFFLINE, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=false 2024-11-14T19:35:18,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T19:35:18,939 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:18,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, ASSIGN because future has completed 2024-11-14T19:35:18,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:35:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T19:35:19,136 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,136 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:35:19,136 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,137 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:35:19,137 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,137 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,148 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,150 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:19,151 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 965abd24f852442dbb51955455ef7db5 columnFamilyName A 2024-11-14T19:35:19,152 DEBUG [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:19,154 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(327): Store=965abd24f852442dbb51955455ef7db5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:19,155 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,158 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:19,159 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 965abd24f852442dbb51955455ef7db5 columnFamilyName B 2024-11-14T19:35:19,159 DEBUG [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:19,160 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(327): Store=965abd24f852442dbb51955455ef7db5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:19,160 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,161 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:19,161 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 965abd24f852442dbb51955455ef7db5 columnFamilyName C 2024-11-14T19:35:19,161 DEBUG [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:19,162 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(327): Store=965abd24f852442dbb51955455ef7db5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:19,162 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,162 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,163 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,163 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,165 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,165 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,166 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:35:19,176 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,179 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:35:19,180 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 965abd24f852442dbb51955455ef7db5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59102449, jitterRate=-0.11930488049983978}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:35:19,180 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,181 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 965abd24f852442dbb51955455ef7db5: Running coprocessor pre-open hook at 1731612919137Writing region info on filesystem at 1731612919137Initializing all the Stores at 1731612919147 (+10 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612919148 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612919148Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612919148Cleaning up temporary data from old regions at 1731612919165 (+17 ms)Running coprocessor post-open hooks at 1731612919180 (+15 ms)Region opened successfully at 1731612919181 (+1 ms) 2024-11-14T19:35:19,183 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., pid=32, masterSystemTime=1731612919098 2024-11-14T19:35:19,187 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=OPEN, openSeqNum=2, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:19,187 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,187 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:35:19,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-14T19:35:19,197 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 in 252 msec 2024-11-14T19:35:19,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-14T19:35:19,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, ASSIGN in 412 msec 2024-11-14T19:35:19,201 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T19:35:19,202 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612919201"}]},"ts":"1731612919201"} 2024-11-14T19:35:19,204 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T19:35:19,205 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T19:35:19,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 895 msec 2024-11-14T19:35:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-14T19:35:19,454 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:19,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2bb5eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:19,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:19,456 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:19,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:19,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc28048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:19,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:19,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:19,465 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366f7c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:19,472 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:19,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:19,477 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:19,479 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T19:35:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 867b237d0fa7,38019,1731612888274 2024-11-14T19:35:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@678171c4 2024-11-14T19:35:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T19:35:19,483 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T19:35:19,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:35:19,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:35:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:19,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741941_1117 (size=1097) 2024-11-14T19:35:19,543 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T19:35:19,543 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T19:35:19,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:35:19,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, REOPEN/MOVE}] 2024-11-14T19:35:19,565 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, REOPEN/MOVE 2024-11-14T19:35:19,566 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:19,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, REOPEN/MOVE because future has completed 2024-11-14T19:35:19,570 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:35:19,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:35:19,724 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,725 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:35:19,725 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 965abd24f852442dbb51955455ef7db5, disabling compactions & flushes 2024-11-14T19:35:19,725 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,725 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,725 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. after waiting 0 ms 2024-11-14T19:35:19,725 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,744 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T19:35:19,745 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:19,745 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 965abd24f852442dbb51955455ef7db5: Waiting for close lock at 1731612919725Running coprocessor pre-close hooks at 1731612919725Disabling compacts and flushes for region at 1731612919725Disabling writes for close at 1731612919725Writing region close event to WAL at 1731612919725Running coprocessor post-close hooks at 1731612919745 (+20 ms)Closed at 1731612919745 2024-11-14T19:35:19,746 WARN [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 965abd24f852442dbb51955455ef7db5 to self. 2024-11-14T19:35:19,750 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:19,750 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=CLOSED 2024-11-14T19:35:19,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:35:19,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-14T19:35:19,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 in 202 msec 2024-11-14T19:35:19,776 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, REOPEN/MOVE; state=CLOSED, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=true 2024-11-14T19:35:19,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:19,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, REOPEN/MOVE because future has completed 2024-11-14T19:35:19,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:35:20,090 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,090 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:35:20,091 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,091 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:35:20,091 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,091 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,095 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,100 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:20,114 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 965abd24f852442dbb51955455ef7db5 columnFamilyName A 2024-11-14T19:35:20,116 DEBUG [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:20,118 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(327): Store=965abd24f852442dbb51955455ef7db5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:20,118 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,119 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:20,119 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 965abd24f852442dbb51955455ef7db5 columnFamilyName B 2024-11-14T19:35:20,119 DEBUG [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:20,121 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(327): Store=965abd24f852442dbb51955455ef7db5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:20,121 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,122 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:20,123 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 965abd24f852442dbb51955455ef7db5 columnFamilyName C 2024-11-14T19:35:20,123 DEBUG [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:20,124 INFO [StoreOpener-965abd24f852442dbb51955455ef7db5-1 {}] regionserver.HStore(327): Store=965abd24f852442dbb51955455ef7db5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:20,124 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,124 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,126 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,127 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,129 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,129 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,130 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:35:20,131 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,133 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 965abd24f852442dbb51955455ef7db5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59070530, jitterRate=-0.1197805106639862}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:35:20,133 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,134 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 965abd24f852442dbb51955455ef7db5: Running coprocessor pre-open hook at 1731612920091Writing region info on filesystem at 1731612920091Initializing all the Stores at 1731612920092 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612920092Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612920093 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612920093Cleaning up temporary data from old regions at 1731612920129 (+36 ms)Running coprocessor post-open hooks at 1731612920133 (+4 ms)Region opened successfully at 1731612920134 (+1 ms) 2024-11-14T19:35:20,135 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., pid=37, masterSystemTime=1731612920084 2024-11-14T19:35:20,142 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=OPEN, openSeqNum=5, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,145 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,145 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:35:20,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-14T19:35:20,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 in 216 msec 2024-11-14T19:35:20,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-14T19:35:20,165 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, REOPEN/MOVE in 591 msec 2024-11-14T19:35:20,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-14T19:35:20,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 619 msec 2024-11-14T19:35:20,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 678 msec 2024-11-14T19:35:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-14T19:35:20,180 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:20,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e01494b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,182 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c765914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,185 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f5c30e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28944b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,189 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d54e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,193 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@220a12f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a95d199, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,208 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7be54787, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,211 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488d94b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e51468f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,221 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec1afe3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,223 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fba4768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1fcab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,240 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd5748f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,243 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49d4b1c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337f1b21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,247 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b69c6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,253 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be0bb19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62885e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,257 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e814bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,260 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3448488d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138dc9c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,263 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75cf965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,266 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc3611c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c264dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:20,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:20,272 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31121a98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:20,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:20,274 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad43a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:20,279 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,295 DEBUG [Thread-600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,295 DEBUG [Thread-596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,305 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,305 DEBUG [Thread-604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:20,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,309 DEBUG [Thread-596 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-14T19:35:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T19:35:20,310 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:20,311 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:20,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,312 DEBUG [Thread-596 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,321 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,334 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,339 DEBUG [Thread-598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:20,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,369 DEBUG [Thread-602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:20,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:20,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T19:35:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:35:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:20,386 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:20,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T19:35:20,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111487384021877e477599f3be2f55cf7d32_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612920377/Put/seqid=0 2024-11-14T19:35:20,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612980459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612980459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612980459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741942_1118 (size=12154) 2024-11-14T19:35:20,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612980480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612980481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,508 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T19:35:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,517 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111487384021877e477599f3be2f55cf7d32_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487384021877e477599f3be2f55cf7d32_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:20,521 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/690e2996f3674489a4812ccb0a8e5df5, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:20,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/690e2996f3674489a4812ccb0a8e5df5 is 175, key is test_row_0/A:col10/1731612920377/Put/seqid=0 2024-11-14T19:35:20,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741943_1119 (size=30951) 2024-11-14T19:35:20,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612980583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612980583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612980584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612980613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612980613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T19:35:20,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T19:35:20,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:20,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612980793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612980794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T19:35:20,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:20,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612980803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612980835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:20,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612980835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:20,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:20,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T19:35:20,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T19:35:20,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:20,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:20,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:20,981 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/690e2996f3674489a4812ccb0a8e5df5 2024-11-14T19:35:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:21,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/480f92d0e2294bd9a6a2e6bb62b7a416 is 50, key is test_row_0/B:col10/1731612920377/Put/seqid=0 2024-11-14T19:35:21,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741944_1120 (size=9657) 2024-11-14T19:35:21,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/480f92d0e2294bd9a6a2e6bb62b7a416 2024-11-14T19:35:21,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612981114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612981114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T19:35:21,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612981134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:21,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:21,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:21,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a7d0d76bf4a24d09b908b6d3e845ce74 is 50, key is test_row_0/C:col10/1731612920377/Put/seqid=0 2024-11-14T19:35:21,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612981155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612981155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741945_1121 (size=9657) 2024-11-14T19:35:21,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a7d0d76bf4a24d09b908b6d3e845ce74 2024-11-14T19:35:21,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/690e2996f3674489a4812ccb0a8e5df5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/690e2996f3674489a4812ccb0a8e5df5 2024-11-14T19:35:21,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/690e2996f3674489a4812ccb0a8e5df5, entries=150, sequenceid=16, filesize=30.2 K 2024-11-14T19:35:21,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/480f92d0e2294bd9a6a2e6bb62b7a416 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/480f92d0e2294bd9a6a2e6bb62b7a416 2024-11-14T19:35:21,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/480f92d0e2294bd9a6a2e6bb62b7a416, entries=100, sequenceid=16, filesize=9.4 K 2024-11-14T19:35:21,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a7d0d76bf4a24d09b908b6d3e845ce74 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a7d0d76bf4a24d09b908b6d3e845ce74 2024-11-14T19:35:21,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a7d0d76bf4a24d09b908b6d3e845ce74, entries=100, sequenceid=16, filesize=9.4 K 2024-11-14T19:35:21,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 965abd24f852442dbb51955455ef7db5 in 845ms, sequenceid=16, compaction requested=false 2024-11-14T19:35:21,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:21,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-14T19:35:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:21,299 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:35:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:21,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d53f20c527bf425e9beb4982075d2cf4_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612920474/Put/seqid=0 2024-11-14T19:35:21,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741946_1122 (size=12154) 2024-11-14T19:35:21,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:21,375 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d53f20c527bf425e9beb4982075d2cf4_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d53f20c527bf425e9beb4982075d2cf4_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:21,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/7e654dbfaf5e48e883edfa384d14e286, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/7e654dbfaf5e48e883edfa384d14e286 is 175, key is test_row_0/A:col10/1731612920474/Put/seqid=0 2024-11-14T19:35:21,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741947_1123 (size=30955) 2024-11-14T19:35:21,415 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/7e654dbfaf5e48e883edfa384d14e286 2024-11-14T19:35:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/bff7948da738494395d535893c9170c2 is 50, key is test_row_0/B:col10/1731612920474/Put/seqid=0 2024-11-14T19:35:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T19:35:21,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741948_1124 (size=12001) 2024-11-14T19:35:21,509 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/bff7948da738494395d535893c9170c2 2024-11-14T19:35:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/22be4069e35747f4ab52a111a1c3e884 is 50, key is test_row_0/C:col10/1731612920474/Put/seqid=0 2024-11-14T19:35:21,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741949_1125 (size=12001) 2024-11-14T19:35:21,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:21,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612981667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612981669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612981670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612981674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612981674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612981784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612981784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612981794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:21,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:21,984 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/22be4069e35747f4ab52a111a1c3e884 2024-11-14T19:35:22,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612981994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612981994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/7e654dbfaf5e48e883edfa384d14e286 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/7e654dbfaf5e48e883edfa384d14e286 2024-11-14T19:35:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,041 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/7e654dbfaf5e48e883edfa384d14e286, entries=150, sequenceid=36, filesize=30.2 K 2024-11-14T19:35:22,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612982035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/bff7948da738494395d535893c9170c2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/bff7948da738494395d535893c9170c2 2024-11-14T19:35:22,063 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/bff7948da738494395d535893c9170c2, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T19:35:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/22be4069e35747f4ab52a111a1c3e884 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/22be4069e35747f4ab52a111a1c3e884 2024-11-14T19:35:22,076 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/22be4069e35747f4ab52a111a1c3e884, entries=150, sequenceid=36, filesize=11.7 K 2024-11-14T19:35:22,079 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 965abd24f852442dbb51955455ef7db5 in 780ms, sequenceid=36, compaction requested=false 2024-11-14T19:35:22,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:22,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:22,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-14T19:35:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-14T19:35:22,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-14T19:35:22,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7370 sec 2024-11-14T19:35:22,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.7770 sec 2024-11-14T19:35:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:22,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:35:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:22,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:22,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147264622641f743468c1485adc1381992_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612921665/Put/seqid=0 2024-11-14T19:35:22,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741950_1126 (size=14594) 2024-11-14T19:35:22,433 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:22,440 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147264622641f743468c1485adc1381992_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147264622641f743468c1485adc1381992_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:22,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/fa2b1c85fc8944d0bac2164fc6092010, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:22,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/fa2b1c85fc8944d0bac2164fc6092010 is 175, key is test_row_0/A:col10/1731612921665/Put/seqid=0 2024-11-14T19:35:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-14T19:35:22,454 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:22,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-14T19:35:22,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612982440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612982443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,458 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:22,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612982443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,459 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:22,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T19:35:22,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741951_1127 (size=39549) 2024-11-14T19:35:22,499 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/fa2b1c85fc8944d0bac2164fc6092010 2024-11-14T19:35:22,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/65d46f543cba4514bb4968e844c3ce8d is 50, key is test_row_0/B:col10/1731612921665/Put/seqid=0 2024-11-14T19:35:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T19:35:22,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612982564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612982564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612982564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T19:35:22,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:22,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:22,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741952_1128 (size=12001) 2024-11-14T19:35:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612982694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,698 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612982694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,700 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T19:35:22,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T19:35:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:22,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612982784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612982784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612982784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:22,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:22,917 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T19:35:22,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T19:35:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:22,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:23,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/65d46f543cba4514bb4968e844c3ce8d 2024-11-14T19:35:23,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/37c2b2a1fc1345bd8d15104f431801e3 is 50, key is test_row_0/C:col10/1731612921665/Put/seqid=0 2024-11-14T19:35:23,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T19:35:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:23,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:23,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T19:35:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:23,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741953_1129 (size=12001) 2024-11-14T19:35:23,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/37c2b2a1fc1345bd8d15104f431801e3 2024-11-14T19:35:23,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612983094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612983094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612983094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/fa2b1c85fc8944d0bac2164fc6092010 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/fa2b1c85fc8944d0bac2164fc6092010 2024-11-14T19:35:23,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/fa2b1c85fc8944d0bac2164fc6092010, entries=200, sequenceid=49, filesize=38.6 K 2024-11-14T19:35:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/65d46f543cba4514bb4968e844c3ce8d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/65d46f543cba4514bb4968e844c3ce8d 2024-11-14T19:35:23,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/65d46f543cba4514bb4968e844c3ce8d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T19:35:23,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/37c2b2a1fc1345bd8d15104f431801e3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/37c2b2a1fc1345bd8d15104f431801e3 2024-11-14T19:35:23,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/37c2b2a1fc1345bd8d15104f431801e3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-14T19:35:23,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 965abd24f852442dbb51955455ef7db5 in 783ms, sequenceid=49, compaction requested=true 2024-11-14T19:35:23,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:23,135 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:23,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:23,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:23,136 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:23,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:23,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:23,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:23,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:23,137 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:23,137 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:23,137 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:23,137 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:23,138 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:23,138 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:23,138 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/690e2996f3674489a4812ccb0a8e5df5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/7e654dbfaf5e48e883edfa384d14e286, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/fa2b1c85fc8944d0bac2164fc6092010] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=99.1 K 2024-11-14T19:35:23,138 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/480f92d0e2294bd9a6a2e6bb62b7a416, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/bff7948da738494395d535893c9170c2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/65d46f543cba4514bb4968e844c3ce8d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=32.9 K 2024-11-14T19:35:23,138 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:23,138 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/690e2996f3674489a4812ccb0a8e5df5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/7e654dbfaf5e48e883edfa384d14e286, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/fa2b1c85fc8944d0bac2164fc6092010] 2024-11-14T19:35:23,139 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 480f92d0e2294bd9a6a2e6bb62b7a416, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731612920370 2024-11-14T19:35:23,139 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 690e2996f3674489a4812ccb0a8e5df5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731612920369 2024-11-14T19:35:23,140 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting bff7948da738494395d535893c9170c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731612920451 2024-11-14T19:35:23,140 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e654dbfaf5e48e883edfa384d14e286, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731612920451 2024-11-14T19:35:23,140 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d46f543cba4514bb4968e844c3ce8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731612921665 2024-11-14T19:35:23,140 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa2b1c85fc8944d0bac2164fc6092010, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731612921665 2024-11-14T19:35:23,165 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:23,165 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/73e13923d6254283b5ad07b434b37304 is 50, key is test_row_0/B:col10/1731612921665/Put/seqid=0 2024-11-14T19:35:23,174 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:23,196 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111468a1d779afa24ad7bcb44fd84ed89d5e_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:23,204 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111468a1d779afa24ad7bcb44fd84ed89d5e_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:23,205 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111468a1d779afa24ad7bcb44fd84ed89d5e_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:23,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741954_1130 (size=12104) 2024-11-14T19:35:23,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-14T19:35:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:23,237 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:35:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:23,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:23,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741955_1131 (size=4469) 2024-11-14T19:35:23,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146409bfa8ff9648cfb7e4b8997df4c942_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612922439/Put/seqid=0 2024-11-14T19:35:23,310 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#112 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:23,311 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/271588a7aaf74c0e94e301b244411c5f is 175, key is test_row_0/A:col10/1731612921665/Put/seqid=0 2024-11-14T19:35:23,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741956_1132 (size=12154) 2024-11-14T19:35:23,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:23,387 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146409bfa8ff9648cfb7e4b8997df4c942_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146409bfa8ff9648cfb7e4b8997df4c942_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:23,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/d34d6f13128445b7b08f890d4fef2f48, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:23,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/d34d6f13128445b7b08f890d4fef2f48 is 175, key is test_row_0/A:col10/1731612922439/Put/seqid=0 2024-11-14T19:35:23,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741957_1133 (size=31058) 2024-11-14T19:35:23,406 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/271588a7aaf74c0e94e301b244411c5f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/271588a7aaf74c0e94e301b244411c5f 2024-11-14T19:35:23,417 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into 271588a7aaf74c0e94e301b244411c5f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:23,417 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:23,418 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=13, startTime=1731612923135; duration=0sec 2024-11-14T19:35:23,418 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:23,419 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:23,419 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:23,422 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:23,422 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:23,422 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:23,422 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a7d0d76bf4a24d09b908b6d3e845ce74, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/22be4069e35747f4ab52a111a1c3e884, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/37c2b2a1fc1345bd8d15104f431801e3] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=32.9 K 2024-11-14T19:35:23,423 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d0d76bf4a24d09b908b6d3e845ce74, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731612920370 2024-11-14T19:35:23,423 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22be4069e35747f4ab52a111a1c3e884, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731612920451 2024-11-14T19:35:23,424 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37c2b2a1fc1345bd8d15104f431801e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731612921665 2024-11-14T19:35:23,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741958_1134 (size=30955) 2024-11-14T19:35:23,437 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/d34d6f13128445b7b08f890d4fef2f48 2024-11-14T19:35:23,446 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:23,447 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/e96b0406fc704087b141ba500a1dcbfa is 50, key is test_row_0/C:col10/1731612921665/Put/seqid=0 2024-11-14T19:35:23,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/1bcdcb2ae8ff4771be45bf568a2475d9 is 50, key is test_row_0/B:col10/1731612922439/Put/seqid=0 2024-11-14T19:35:23,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741959_1135 (size=12104) 2024-11-14T19:35:23,494 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/e96b0406fc704087b141ba500a1dcbfa as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/e96b0406fc704087b141ba500a1dcbfa 2024-11-14T19:35:23,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741960_1136 (size=12001) 2024-11-14T19:35:23,515 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/1bcdcb2ae8ff4771be45bf568a2475d9 2024-11-14T19:35:23,521 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into e96b0406fc704087b141ba500a1dcbfa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:23,522 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:23,522 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=13, startTime=1731612923136; duration=0sec 2024-11-14T19:35:23,522 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:23,522 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:23,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/4afe457a5d144acc8e228bd230ed25d5 is 50, key is test_row_0/C:col10/1731612922439/Put/seqid=0 2024-11-14T19:35:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T19:35:23,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:23,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741961_1137 (size=12001) 2024-11-14T19:35:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:23,642 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/73e13923d6254283b5ad07b434b37304 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/73e13923d6254283b5ad07b434b37304 2024-11-14T19:35:23,650 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into 73e13923d6254283b5ad07b434b37304(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:23,650 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:23,650 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=13, startTime=1731612923136; duration=0sec 2024-11-14T19:35:23,651 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:23,651 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:23,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612983657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612983663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612983667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612983774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612983774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612983774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612983984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612983985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:23,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612983984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:23,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:23,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,018 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/4afe457a5d144acc8e228bd230ed25d5 2024-11-14T19:35:24,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/d34d6f13128445b7b08f890d4fef2f48 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/d34d6f13128445b7b08f890d4fef2f48 2024-11-14T19:35:24,036 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/d34d6f13128445b7b08f890d4fef2f48, entries=150, sequenceid=69, filesize=30.2 K 2024-11-14T19:35:24,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/1bcdcb2ae8ff4771be45bf568a2475d9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/1bcdcb2ae8ff4771be45bf568a2475d9 2024-11-14T19:35:24,046 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/1bcdcb2ae8ff4771be45bf568a2475d9, entries=150, sequenceid=69, filesize=11.7 K 2024-11-14T19:35:24,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/4afe457a5d144acc8e228bd230ed25d5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/4afe457a5d144acc8e228bd230ed25d5 2024-11-14T19:35:24,064 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/4afe457a5d144acc8e228bd230ed25d5, entries=150, sequenceid=69, filesize=11.7 K 2024-11-14T19:35:24,067 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 965abd24f852442dbb51955455ef7db5 in 830ms, sequenceid=69, compaction requested=false 2024-11-14T19:35:24,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:24,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-14T19:35:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-14T19:35:24,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-14T19:35:24,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6100 sec 2024-11-14T19:35:24,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6180 sec 2024-11-14T19:35:24,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T19:35:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:24,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:24,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:24,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:24,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:24,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:24,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:24,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bfcf86e33f5342579c93607bf03d4cb3_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612924309/Put/seqid=0 2024-11-14T19:35:24,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612984368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612984371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612984374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741962_1138 (size=12154) 2024-11-14T19:35:24,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:24,426 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114bfcf86e33f5342579c93607bf03d4cb3_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bfcf86e33f5342579c93607bf03d4cb3_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:24,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f000a6e0baff4bb487c153ffa8daa4c6, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:24,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f000a6e0baff4bb487c153ffa8daa4c6 is 175, key is test_row_0/A:col10/1731612924309/Put/seqid=0 2024-11-14T19:35:24,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741963_1139 (size=30955) 2024-11-14T19:35:24,464 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f000a6e0baff4bb487c153ffa8daa4c6 2024-11-14T19:35:24,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f107aeb88e8344c989c684431cb30125 is 50, key is test_row_0/B:col10/1731612924309/Put/seqid=0 2024-11-14T19:35:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612984484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612984485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612984485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741964_1140 (size=12001) 2024-11-14T19:35:24,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f107aeb88e8344c989c684431cb30125 2024-11-14T19:35:24,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b7311b53f95b404ab4d1805d1193f843 is 50, key is test_row_0/C:col10/1731612924309/Put/seqid=0 2024-11-14T19:35:24,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741965_1141 (size=12001) 2024-11-14T19:35:24,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b7311b53f95b404ab4d1805d1193f843 2024-11-14T19:35:24,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f000a6e0baff4bb487c153ffa8daa4c6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f000a6e0baff4bb487c153ffa8daa4c6 2024-11-14T19:35:24,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f000a6e0baff4bb487c153ffa8daa4c6, entries=150, sequenceid=88, filesize=30.2 K 2024-11-14T19:35:24,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f107aeb88e8344c989c684431cb30125 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f107aeb88e8344c989c684431cb30125 2024-11-14T19:35:24,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f107aeb88e8344c989c684431cb30125, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T19:35:24,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b7311b53f95b404ab4d1805d1193f843 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7311b53f95b404ab4d1805d1193f843 2024-11-14T19:35:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-14T19:35:24,604 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:24,605 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-14T19:35:24,608 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T19:35:24,610 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:24,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:24,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7311b53f95b404ab4d1805d1193f843, entries=150, sequenceid=88, filesize=11.7 K 2024-11-14T19:35:24,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 965abd24f852442dbb51955455ef7db5 in 313ms, sequenceid=88, compaction requested=true 2024-11-14T19:35:24,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:24,625 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:24,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:24,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:24,627 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:24,627 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:24,627 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:24,627 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,627 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/271588a7aaf74c0e94e301b244411c5f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/d34d6f13128445b7b08f890d4fef2f48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f000a6e0baff4bb487c153ffa8daa4c6] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=90.8 K 2024-11-14T19:35:24,628 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,628 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/271588a7aaf74c0e94e301b244411c5f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/d34d6f13128445b7b08f890d4fef2f48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f000a6e0baff4bb487c153ffa8daa4c6] 2024-11-14T19:35:24,628 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 271588a7aaf74c0e94e301b244411c5f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731612921665 2024-11-14T19:35:24,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:24,632 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d34d6f13128445b7b08f890d4fef2f48, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731612922422 2024-11-14T19:35:24,633 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting f000a6e0baff4bb487c153ffa8daa4c6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731612923657 2024-11-14T19:35:24,633 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:24,634 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:24,634 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,634 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/73e13923d6254283b5ad07b434b37304, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/1bcdcb2ae8ff4771be45bf568a2475d9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f107aeb88e8344c989c684431cb30125] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=35.3 K 2024-11-14T19:35:24,635 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 73e13923d6254283b5ad07b434b37304, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731612921665 2024-11-14T19:35:24,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:24,635 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bcdcb2ae8ff4771be45bf568a2475d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731612922422 2024-11-14T19:35:24,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:24,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:24,636 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f107aeb88e8344c989c684431cb30125, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731612923657 2024-11-14T19:35:24,662 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:24,677 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:24,678 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/4f0f7e81c97e4e6a812480cf481feaeb is 50, key is test_row_0/B:col10/1731612924309/Put/seqid=0 2024-11-14T19:35:24,682 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411147fb1011f75034ac49c2071b63d65e539_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:24,686 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411147fb1011f75034ac49c2071b63d65e539_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:24,686 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147fb1011f75034ac49c2071b63d65e539_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:24,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T19:35:24,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:24,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:24,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:24,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T19:35:24,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741966_1142 (size=12207) 2024-11-14T19:35:24,745 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/4f0f7e81c97e4e6a812480cf481feaeb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/4f0f7e81c97e4e6a812480cf481feaeb 2024-11-14T19:35:24,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612984729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612984738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612984741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612984749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612984750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741967_1143 (size=4469) 2024-11-14T19:35:24,765 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into 4f0f7e81c97e4e6a812480cf481feaeb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:24,766 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#120 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:24,766 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/69aec367d19e450bbffd86cbc3443e4f is 175, key is test_row_0/A:col10/1731612924309/Put/seqid=0 2024-11-14T19:35:24,766 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:24,767 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=13, startTime=1731612924626; duration=0sec 2024-11-14T19:35:24,767 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:24,767 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:24,767 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:24,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:24,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:24,770 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:24,770 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:24,771 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,771 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/e96b0406fc704087b141ba500a1dcbfa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/4afe457a5d144acc8e228bd230ed25d5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7311b53f95b404ab4d1805d1193f843] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=35.3 K 2024-11-14T19:35:24,772 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting e96b0406fc704087b141ba500a1dcbfa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731612921665 2024-11-14T19:35:24,773 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4afe457a5d144acc8e228bd230ed25d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731612922422 2024-11-14T19:35:24,774 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b7311b53f95b404ab4d1805d1193f843, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731612923657 2024-11-14T19:35:24,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148e643249e9894c5f82b4d41630270d87_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612924694/Put/seqid=0 2024-11-14T19:35:24,833 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:24,834 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b49f49b5395d418cb16bc8e29a982624 is 50, key is test_row_0/C:col10/1731612924309/Put/seqid=0 2024-11-14T19:35:24,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612984854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612984854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612984854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741968_1144 (size=31161) 2024-11-14T19:35:24,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612984863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612984863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:24,875 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/69aec367d19e450bbffd86cbc3443e4f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/69aec367d19e450bbffd86cbc3443e4f 2024-11-14T19:35:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:24,883 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into 69aec367d19e450bbffd86cbc3443e4f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:24,883 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:24,883 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=13, startTime=1731612924625; duration=0sec 2024-11-14T19:35:24,883 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:24,883 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:24,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T19:35:24,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741969_1145 (size=14594) 2024-11-14T19:35:24,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:24,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:24,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:24,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:24,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741970_1146 (size=12207) 2024-11-14T19:35:24,970 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b49f49b5395d418cb16bc8e29a982624 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b49f49b5395d418cb16bc8e29a982624 2024-11-14T19:35:24,978 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into b49f49b5395d418cb16bc8e29a982624(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:24,978 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:24,978 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=13, startTime=1731612924635; duration=0sec 2024-11-14T19:35:24,978 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:24,978 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:25,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:25,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:25,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612985073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612985073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612985073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,089 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612985084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612985084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T19:35:25,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:25,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:25,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,326 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:25,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148e643249e9894c5f82b4d41630270d87_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148e643249e9894c5f82b4d41630270d87_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:25,335 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/08ab3c18d18b4f2787abebf06ca00afe, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:25,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/08ab3c18d18b4f2787abebf06ca00afe is 175, key is test_row_0/A:col10/1731612924694/Put/seqid=0 2024-11-14T19:35:25,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612985394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741971_1147 (size=39549) 2024-11-14T19:35:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612985394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,408 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=105, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/08ab3c18d18b4f2787abebf06ca00afe 2024-11-14T19:35:25,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612985407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612985407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612985407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/c6f2d711f86946eebe2cf3720edee464 is 50, key is test_row_0/B:col10/1731612924694/Put/seqid=0 2024-11-14T19:35:25,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741972_1148 (size=12001) 2024-11-14T19:35:25,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:25,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:25,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T19:35:25,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:25,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:25,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:25,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:25,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/c6f2d711f86946eebe2cf3720edee464 2024-11-14T19:35:25,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/199cf73a0cd84bc7b913ed737a10bd4b is 50, key is test_row_0/C:col10/1731612924694/Put/seqid=0 2024-11-14T19:35:25,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612985913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,922 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612985914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612985923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612985923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612985924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:25,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:25,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:25,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741973_1149 (size=12001) 2024-11-14T19:35:25,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/199cf73a0cd84bc7b913ed737a10bd4b 2024-11-14T19:35:25,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/08ab3c18d18b4f2787abebf06ca00afe as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/08ab3c18d18b4f2787abebf06ca00afe 2024-11-14T19:35:25,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/08ab3c18d18b4f2787abebf06ca00afe, entries=200, sequenceid=105, filesize=38.6 K 2024-11-14T19:35:25,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/c6f2d711f86946eebe2cf3720edee464 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/c6f2d711f86946eebe2cf3720edee464 2024-11-14T19:35:26,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/c6f2d711f86946eebe2cf3720edee464, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T19:35:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/199cf73a0cd84bc7b913ed737a10bd4b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/199cf73a0cd84bc7b913ed737a10bd4b 2024-11-14T19:35:26,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/199cf73a0cd84bc7b913ed737a10bd4b, entries=150, sequenceid=105, filesize=11.7 K 2024-11-14T19:35:26,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 965abd24f852442dbb51955455ef7db5 in 1298ms, sequenceid=105, compaction requested=false 2024-11-14T19:35:26,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:26,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-14T19:35:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:26,025 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:35:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:26,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:26,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148109706809754114914be26bf5a48809_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612924735/Put/seqid=0 2024-11-14T19:35:26,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741974_1150 (size=12154) 2024-11-14T19:35:26,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:26,144 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148109706809754114914be26bf5a48809_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148109706809754114914be26bf5a48809_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:26,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5325de52c9814776b4a2b616b9aca6d4, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:26,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5325de52c9814776b4a2b616b9aca6d4 is 175, key is test_row_0/A:col10/1731612924735/Put/seqid=0 2024-11-14T19:35:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741975_1151 (size=30955) 2024-11-14T19:35:26,209 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5325de52c9814776b4a2b616b9aca6d4 2024-11-14T19:35:26,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/a5c5a32df7a04add8a83b6cf53bd44ca is 50, key is test_row_0/B:col10/1731612924735/Put/seqid=0 2024-11-14T19:35:26,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741976_1152 (size=12001) 2024-11-14T19:35:26,675 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/a5c5a32df7a04add8a83b6cf53bd44ca 2024-11-14T19:35:26,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/8759b9faea804975b3ed3dccd60491f5 is 50, key is test_row_0/C:col10/1731612924735/Put/seqid=0 2024-11-14T19:35:26,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741977_1153 (size=12001) 2024-11-14T19:35:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T19:35:26,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:26,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612986967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:26,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612986968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612986969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:26,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612986976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612986978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612987083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612987083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612987084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612987093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612987093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,131 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/8759b9faea804975b3ed3dccd60491f5 2024-11-14T19:35:27,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5325de52c9814776b4a2b616b9aca6d4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5325de52c9814776b4a2b616b9aca6d4 2024-11-14T19:35:27,161 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5325de52c9814776b4a2b616b9aca6d4, entries=150, sequenceid=123, filesize=30.2 K 2024-11-14T19:35:27,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/a5c5a32df7a04add8a83b6cf53bd44ca as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a5c5a32df7a04add8a83b6cf53bd44ca 2024-11-14T19:35:27,182 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a5c5a32df7a04add8a83b6cf53bd44ca, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T19:35:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/8759b9faea804975b3ed3dccd60491f5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8759b9faea804975b3ed3dccd60491f5 2024-11-14T19:35:27,212 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8759b9faea804975b3ed3dccd60491f5, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T19:35:27,214 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 965abd24f852442dbb51955455ef7db5 in 1188ms, sequenceid=123, compaction requested=true 2024-11-14T19:35:27,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:27,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:27,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-14T19:35:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-14T19:35:27,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-14T19:35:27,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6060 sec 2024-11-14T19:35:27,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.6140 sec 2024-11-14T19:35:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:27,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:35:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:27,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:27,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d2feb3b15e014f1fbce0d4cd6607ff58_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612926976/Put/seqid=0 2024-11-14T19:35:27,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612987323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612987320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612987331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612987334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612987334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741978_1154 (size=12304) 2024-11-14T19:35:27,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612987444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612987444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612987444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612987445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612987445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612987657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612987657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612987657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612987657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612987657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,754 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:27,765 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114d2feb3b15e014f1fbce0d4cd6607ff58_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d2feb3b15e014f1fbce0d4cd6607ff58_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:27,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5bf40b7408334d05bfb4fdb53bfc72e4, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:27,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5bf40b7408334d05bfb4fdb53bfc72e4 is 175, key is test_row_0/A:col10/1731612926976/Put/seqid=0 2024-11-14T19:35:27,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741979_1155 (size=31105) 2024-11-14T19:35:27,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612987974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612987974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612987974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612987974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:27,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612987974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:27,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:28,209 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5bf40b7408334d05bfb4fdb53bfc72e4 2024-11-14T19:35:28,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/523b9fccc41846e98d2befa1fa991ec7 is 50, key is test_row_0/B:col10/1731612926976/Put/seqid=0 2024-11-14T19:35:28,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741980_1156 (size=12151) 2024-11-14T19:35:28,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612988483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:28,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612988483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:28,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612988483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:28,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612988484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:28,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612988493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:28,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/523b9fccc41846e98d2befa1fa991ec7 2024-11-14T19:35:28,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/1e5aaffdbfe64858a649cc8dd91674a8 is 50, key is test_row_0/C:col10/1731612926976/Put/seqid=0 2024-11-14T19:35:28,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741981_1157 (size=12151) 2024-11-14T19:35:28,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/1e5aaffdbfe64858a649cc8dd91674a8 2024-11-14T19:35:28,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5bf40b7408334d05bfb4fdb53bfc72e4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5bf40b7408334d05bfb4fdb53bfc72e4 2024-11-14T19:35:28,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5bf40b7408334d05bfb4fdb53bfc72e4, entries=150, sequenceid=142, filesize=30.4 K 2024-11-14T19:35:28,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/523b9fccc41846e98d2befa1fa991ec7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/523b9fccc41846e98d2befa1fa991ec7 2024-11-14T19:35:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-14T19:35:28,784 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:28,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-14T19:35:28,787 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:28,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/523b9fccc41846e98d2befa1fa991ec7, entries=150, sequenceid=142, filesize=11.9 K 2024-11-14T19:35:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T19:35:28,789 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:28,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:28,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/1e5aaffdbfe64858a649cc8dd91674a8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/1e5aaffdbfe64858a649cc8dd91674a8 2024-11-14T19:35:28,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/1e5aaffdbfe64858a649cc8dd91674a8, entries=150, sequenceid=142, filesize=11.9 K 2024-11-14T19:35:28,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 965abd24f852442dbb51955455ef7db5 in 1504ms, sequenceid=142, compaction requested=true 2024-11-14T19:35:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:28,811 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:28,811 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:28,813 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:28,813 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:28,813 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:28,813 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/69aec367d19e450bbffd86cbc3443e4f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/08ab3c18d18b4f2787abebf06ca00afe, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5325de52c9814776b4a2b616b9aca6d4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5bf40b7408334d05bfb4fdb53bfc72e4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=129.7 K 2024-11-14T19:35:28,813 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:28,813 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/69aec367d19e450bbffd86cbc3443e4f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/08ab3c18d18b4f2787abebf06ca00afe, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5325de52c9814776b4a2b616b9aca6d4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5bf40b7408334d05bfb4fdb53bfc72e4] 2024-11-14T19:35:28,815 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:28,815 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69aec367d19e450bbffd86cbc3443e4f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731612923657 2024-11-14T19:35:28,815 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:28,815 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:28,815 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/4f0f7e81c97e4e6a812480cf481feaeb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/c6f2d711f86946eebe2cf3720edee464, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a5c5a32df7a04add8a83b6cf53bd44ca, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/523b9fccc41846e98d2befa1fa991ec7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=47.2 K 2024-11-14T19:35:28,816 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08ab3c18d18b4f2787abebf06ca00afe, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731612924341 2024-11-14T19:35:28,816 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f0f7e81c97e4e6a812480cf481feaeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731612923657 2024-11-14T19:35:28,816 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5325de52c9814776b4a2b616b9aca6d4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731612924735 2024-11-14T19:35:28,816 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f2d711f86946eebe2cf3720edee464, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731612924368 2024-11-14T19:35:28,816 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf40b7408334d05bfb4fdb53bfc72e4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731612926949 2024-11-14T19:35:28,817 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c5a32df7a04add8a83b6cf53bd44ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731612924735 2024-11-14T19:35:28,818 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 523b9fccc41846e98d2befa1fa991ec7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731612926949 2024-11-14T19:35:28,861 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:28,862 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/9f756315bd324eebbd11a51cc6a4773f is 50, key is test_row_0/B:col10/1731612926976/Put/seqid=0 2024-11-14T19:35:28,888 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T19:35:28,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T19:35:28,897 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T19:35:28,909 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114f59ab616d7ce46868339f4a518fa206e_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:28,912 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114f59ab616d7ce46868339f4a518fa206e_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:28,915 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f59ab616d7ce46868339f4a518fa206e_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:28,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741982_1158 (size=12493) 2024-11-14T19:35:28,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-14T19:35:28,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:28,944 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:35:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:28,959 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/9f756315bd324eebbd11a51cc6a4773f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9f756315bd324eebbd11a51cc6a4773f 2024-11-14T19:35:28,966 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into 9f756315bd324eebbd11a51cc6a4773f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:28,966 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:28,966 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=12, startTime=1731612928811; duration=0sec 2024-11-14T19:35:28,967 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:28,967 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:28,967 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:28,969 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:28,969 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:28,969 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:28,969 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b49f49b5395d418cb16bc8e29a982624, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/199cf73a0cd84bc7b913ed737a10bd4b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8759b9faea804975b3ed3dccd60491f5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/1e5aaffdbfe64858a649cc8dd91674a8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=47.2 K 2024-11-14T19:35:28,970 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b49f49b5395d418cb16bc8e29a982624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731612923657 2024-11-14T19:35:28,970 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 199cf73a0cd84bc7b913ed737a10bd4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1731612924368 2024-11-14T19:35:28,970 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8759b9faea804975b3ed3dccd60491f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731612924735 2024-11-14T19:35:28,971 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e5aaffdbfe64858a649cc8dd91674a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731612926949 2024-11-14T19:35:29,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741983_1159 (size=4469) 2024-11-14T19:35:29,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144a2f5ccd130743038c436c027b7e55a0_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612927315/Put/seqid=0 2024-11-14T19:35:29,053 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:29,054 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/f630c30195d3481986bb9a9f7e10e9ad is 50, key is test_row_0/C:col10/1731612926976/Put/seqid=0 2024-11-14T19:35:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T19:35:29,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741984_1160 (size=12304) 2024-11-14T19:35:29,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741985_1161 (size=12493) 2024-11-14T19:35:29,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,168 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144a2f5ccd130743038c436c027b7e55a0_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144a2f5ccd130743038c436c027b7e55a0_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5098ed22dd194f7d91c9d4d6a7d35101, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:29,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5098ed22dd194f7d91c9d4d6a7d35101 is 175, key is test_row_0/A:col10/1731612927315/Put/seqid=0 2024-11-14T19:35:29,174 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/f630c30195d3481986bb9a9f7e10e9ad as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/f630c30195d3481986bb9a9f7e10e9ad 2024-11-14T19:35:29,188 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into f630c30195d3481986bb9a9f7e10e9ad(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:29,188 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:29,188 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=12, startTime=1731612928811; duration=0sec 2024-11-14T19:35:29,188 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:29,189 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:29,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741986_1162 (size=31105) 2024-11-14T19:35:29,222 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5098ed22dd194f7d91c9d4d6a7d35101 2024-11-14T19:35:29,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/9d75c5e01fed4a7db0ed3c62e57ed452 is 50, key is test_row_0/B:col10/1731612927315/Put/seqid=0 2024-11-14T19:35:29,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741987_1163 (size=12151) 2024-11-14T19:35:29,295 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/9d75c5e01fed4a7db0ed3c62e57ed452 2024-11-14T19:35:29,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9db44299e1c2448cae23299789586960 is 50, key is test_row_0/C:col10/1731612927315/Put/seqid=0 2024-11-14T19:35:29,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741988_1164 (size=12151) 2024-11-14T19:35:29,370 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9db44299e1c2448cae23299789586960 2024-11-14T19:35:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5098ed22dd194f7d91c9d4d6a7d35101 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5098ed22dd194f7d91c9d4d6a7d35101 2024-11-14T19:35:29,382 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5098ed22dd194f7d91c9d4d6a7d35101, entries=150, sequenceid=155, filesize=30.4 K 2024-11-14T19:35:29,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/9d75c5e01fed4a7db0ed3c62e57ed452 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9d75c5e01fed4a7db0ed3c62e57ed452 2024-11-14T19:35:29,389 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9d75c5e01fed4a7db0ed3c62e57ed452, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T19:35:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9db44299e1c2448cae23299789586960 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9db44299e1c2448cae23299789586960 2024-11-14T19:35:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,399 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9db44299e1c2448cae23299789586960, entries=150, sequenceid=155, filesize=11.9 K 2024-11-14T19:35:29,400 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 965abd24f852442dbb51955455ef7db5 in 456ms, sequenceid=155, compaction requested=false 2024-11-14T19:35:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:29,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-14T19:35:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-14T19:35:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-14T19:35:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 613 msec 2024-11-14T19:35:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 621 msec 2024-11-14T19:35:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-14T19:35:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,414 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:29,415 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-14T19:35:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T19:35:29,418 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:29,419 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:29,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,425 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#133 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:29,426 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5db3dce413f14560ad45d4918a59a6e6 is 175, key is test_row_0/A:col10/1731612926976/Put/seqid=0 2024-11-14T19:35:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741989_1165 (size=31447) 2024-11-14T19:35:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T19:35:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-14T19:35:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:29,574 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-14T19:35:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:29,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f0f78936a98045febc79c64d7da6643c_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612929573/Put/seqid=0 2024-11-14T19:35:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741990_1166 (size=7324) 2024-11-14T19:35:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,659 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f0f78936a98045febc79c64d7da6643c_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f0f78936a98045febc79c64d7da6643c_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/3d5213e343d741ab821e793d97bd25b0, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/3d5213e343d741ab821e793d97bd25b0 is 175, key is test_row_0/A:col10/1731612929573/Put/seqid=0 2024-11-14T19:35:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731612929576, max timestamp=1731612929674Num uniques 859; 2024-11-14T19:35:29,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T19:35:29,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731612929576, max timestamp=1731612929674Num uniques 859; 2024-11-14T19:35:29,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T19:35:29,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731612929576, max timestamp=1731612929674Num uniques 859; 2024-11-14T19:35:29,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-14T19:35:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741991_1167 (size=13815) 2024-11-14T19:35:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,709 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/3d5213e343d741ab821e793d97bd25b0 2024-11-14T19:35:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/57d93988cc6043ed92825cd791db5c73 is 50, key is test_row_0/B:col10/1731612929573/Put/seqid=0 2024-11-14T19:35:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T19:35:29,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612989702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612989706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612989733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612989735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612989737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741992_1168 (size=7365) 2024-11-14T19:35:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,777 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/57d93988cc6043ed92825cd791db5c73 2024-11-14T19:35:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:29,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/5af4ace65d574eecba69faee30761bee is 50, key is test_row_0/C:col10/1731612929573/Put/seqid=0 2024-11-14T19:35:29,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612989845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741993_1169 (size=7365) 2024-11-14T19:35:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612989845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612989853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612989853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:29,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612989864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:29,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:29,886 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5db3dce413f14560ad45d4918a59a6e6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5db3dce413f14560ad45d4918a59a6e6 2024-11-14T19:35:29,894 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into 5db3dce413f14560ad45d4918a59a6e6(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-14T19:35:29,894 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:29,894 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=12, startTime=1731612928811; duration=1sec 2024-11-14T19:35:29,895 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:29,895 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T19:35:30,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612990064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612990064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612990064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612990073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612990073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,253 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/5af4ace65d574eecba69faee30761bee 2024-11-14T19:35:30,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/3d5213e343d741ab821e793d97bd25b0 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/3d5213e343d741ab821e793d97bd25b0 2024-11-14T19:35:30,269 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/3d5213e343d741ab821e793d97bd25b0, entries=50, sequenceid=161, filesize=13.5 K 2024-11-14T19:35:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/57d93988cc6043ed92825cd791db5c73 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/57d93988cc6043ed92825cd791db5c73 2024-11-14T19:35:30,288 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/57d93988cc6043ed92825cd791db5c73, entries=50, sequenceid=161, filesize=7.2 K 2024-11-14T19:35:30,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/5af4ace65d574eecba69faee30761bee as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/5af4ace65d574eecba69faee30761bee 2024-11-14T19:35:30,304 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/5af4ace65d574eecba69faee30761bee, entries=50, sequenceid=161, filesize=7.2 K 2024-11-14T19:35:30,305 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=194.56 KB/199230 for 965abd24f852442dbb51955455ef7db5 in 732ms, sequenceid=161, compaction requested=true 2024-11-14T19:35:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:30,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-14T19:35:30,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-14T19:35:30,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-14T19:35:30,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 889 msec 2024-11-14T19:35:30,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 896 msec 2024-11-14T19:35:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:30,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=201.27 KB heapSize=518.58 KB 2024-11-14T19:35:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612990394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612990394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:30,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612990400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612990402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612990405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143b954fdd8380405ba148aa6bd2580960_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612930396/Put/seqid=0 2024-11-14T19:35:30,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741994_1170 (size=17284) 2024-11-14T19:35:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612990524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612990525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612990527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-14T19:35:30,554 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:30,555 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-14T19:35:30,558 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:30,558 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T19:35:30,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T19:35:30,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T19:35:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:30,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:30,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612990734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612990734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612990742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:30,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T19:35:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:30,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:30,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:30,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:30,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T19:35:30,878 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143b954fdd8380405ba148aa6bd2580960_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143b954fdd8380405ba148aa6bd2580960_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:30,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/e021770de2174e09bcb85ef33d752e99, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:30,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/e021770de2174e09bcb85ef33d752e99 is 175, key is test_row_0/A:col10/1731612930396/Put/seqid=0 2024-11-14T19:35:30,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612990903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612990903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:30,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741995_1171 (size=48389) 2024-11-14T19:35:30,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=71.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/e021770de2174e09bcb85ef33d752e99 2024-11-14T19:35:30,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/cf81c55ff6004b74a7cb97578944f7eb is 50, key is test_row_0/B:col10/1731612930396/Put/seqid=0 2024-11-14T19:35:31,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741996_1172 (size=12151) 2024-11-14T19:35:31,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.56 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/cf81c55ff6004b74a7cb97578944f7eb 2024-11-14T19:35:31,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a2b04896158144b2840366d2d32d7ab5 is 50, key is test_row_0/C:col10/1731612930396/Put/seqid=0 2024-11-14T19:35:31,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T19:35:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:31,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612991054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612991054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:31,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612991054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:31,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741997_1173 (size=12151) 2024-11-14T19:35:31,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T19:35:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T19:35:31,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T19:35:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.56 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a2b04896158144b2840366d2d32d7ab5 2024-11-14T19:35:31,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/e021770de2174e09bcb85ef33d752e99 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/e021770de2174e09bcb85ef33d752e99 2024-11-14T19:35:31,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/e021770de2174e09bcb85ef33d752e99, entries=250, sequenceid=197, filesize=47.3 K 2024-11-14T19:35:31,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/cf81c55ff6004b74a7cb97578944f7eb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/cf81c55ff6004b74a7cb97578944f7eb 2024-11-14T19:35:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T19:35:31,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:31,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/cf81c55ff6004b74a7cb97578944f7eb, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T19:35:31,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a2b04896158144b2840366d2d32d7ab5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a2b04896158144b2840366d2d32d7ab5 2024-11-14T19:35:31,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a2b04896158144b2840366d2d32d7ab5, entries=150, sequenceid=197, filesize=11.9 K 2024-11-14T19:35:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~214.69 KB/219840, heapSize ~558.38 KB/571776, currentSize=0 B/0 for 965abd24f852442dbb51955455ef7db5 in 1103ms, sequenceid=197, compaction requested=true 2024-11-14T19:35:31,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:31,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:31,502 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:31,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:31,502 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:31,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:31,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:31,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:31,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,503 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124756 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:31,503 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:31,504 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,504 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5db3dce413f14560ad45d4918a59a6e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5098ed22dd194f7d91c9d4d6a7d35101, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/3d5213e343d741ab821e793d97bd25b0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/e021770de2174e09bcb85ef33d752e99] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=121.8 K 2024-11-14T19:35:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,504 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,504 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5db3dce413f14560ad45d4918a59a6e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5098ed22dd194f7d91c9d4d6a7d35101, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/3d5213e343d741ab821e793d97bd25b0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/e021770de2174e09bcb85ef33d752e99] 2024-11-14T19:35:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,504 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44160 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:31,504 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:31,504 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,505 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9f756315bd324eebbd11a51cc6a4773f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9d75c5e01fed4a7db0ed3c62e57ed452, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/57d93988cc6043ed92825cd791db5c73, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/cf81c55ff6004b74a7cb97578944f7eb] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=43.1 K 2024-11-14T19:35:31,505 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f756315bd324eebbd11a51cc6a4773f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731612926949 2024-11-14T19:35:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,506 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d75c5e01fed4a7db0ed3c62e57ed452, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731612927315 2024-11-14T19:35:31,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,506 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db3dce413f14560ad45d4918a59a6e6, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731612926949 2024-11-14T19:35:31,506 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d93988cc6043ed92825cd791db5c73, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731612929573 2024-11-14T19:35:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,507 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5098ed22dd194f7d91c9d4d6a7d35101, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731612927315 2024-11-14T19:35:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,507 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d5213e343d741ab821e793d97bd25b0, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731612929573 2024-11-14T19:35:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,509 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e021770de2174e09bcb85ef33d752e99, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731612929700 2024-11-14T19:35:31,509 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cf81c55ff6004b74a7cb97578944f7eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731612930396 2024-11-14T19:35:31,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,529 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,553 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#145 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:31,554 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/2dba437973884212b22ae997b23d4f80 is 50, key is test_row_0/B:col10/1731612930396/Put/seqid=0 2024-11-14T19:35:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,557 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411140277efac9ed342549afccf036e1d0bcc_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,563 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411140277efac9ed342549afccf036e1d0bcc_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:31,563 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140277efac9ed342549afccf036e1d0bcc_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741998_1174 (size=12629) 2024-11-14T19:35:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,606 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/2dba437973884212b22ae997b23d4f80 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/2dba437973884212b22ae997b23d4f80 2024-11-14T19:35:31,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,617 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into 2dba437973884212b22ae997b23d4f80(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:31,617 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:31,617 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=12, startTime=1731612931502; duration=0sec 2024-11-14T19:35:31,618 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:31,618 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:31,618 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741999_1175 (size=4469) 2024-11-14T19:35:31,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,624 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44160 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:31,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,625 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:31,625 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,625 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/f630c30195d3481986bb9a9f7e10e9ad, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9db44299e1c2448cae23299789586960, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/5af4ace65d574eecba69faee30761bee, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a2b04896158144b2840366d2d32d7ab5] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=43.1 K 2024-11-14T19:35:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,625 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f630c30195d3481986bb9a9f7e10e9ad, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731612926949 2024-11-14T19:35:31,626 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db44299e1c2448cae23299789586960, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731612927315 2024-11-14T19:35:31,627 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 5af4ace65d574eecba69faee30761bee, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731612929573 2024-11-14T19:35:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,628 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b04896158144b2840366d2d32d7ab5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731612930396 2024-11-14T19:35:31,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-14T19:35:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-14T19:35:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-14T19:35:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-14T19:35:31,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-11-14T19:35:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,659 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#146 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,660 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/89e296bbfb73423d95760f9eb3a043f2 is 50, key is test_row_0/C:col10/1731612930396/Put/seqid=0 2024-11-14T19:35:31,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1040 sec 2024-11-14T19:35:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-14T19:35:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,693 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,694 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-14T19:35:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,698 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T19:35:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,700 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:31,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742000_1176 (size=12629) 2024-11-14T19:35:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,720 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/89e296bbfb73423d95760f9eb3a043f2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/89e296bbfb73423d95760f9eb3a043f2 2024-11-14T19:35:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,728 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into 89e296bbfb73423d95760f9eb3a043f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:31,728 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:31,728 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=12, startTime=1731612931502; duration=0sec 2024-11-14T19:35:31,728 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:31,728 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] io.ByteBuffAllocator(384): Pool already reached its max capacity : 186 and no free buffers now. Consider increasing the value for 'hbase.server.allocator.max.buffer.count' ? 2024-11-14T19:35:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:35:31,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:31,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:31,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:31,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:31,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:31,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T19:35:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114313202b547504142a4212a09b9e125c4_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612931777/Put/seqid=0 2024-11-14T19:35:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T19:35:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:31,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:31,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742001_1177 (size=22268) 2024-11-14T19:35:31,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612931876, max timestamp=1731612931910Num uniques 859; 2024-11-14T19:35:31,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:35:31,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612931876, max timestamp=1731612931910Num uniques 859; 2024-11-14T19:35:31,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:35:31,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612931876, max timestamp=1731612931910Num uniques 859; 2024-11-14T19:35:31,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:35:31,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:31,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731612931976, max timestamp=1731612931993Num uniques 1018; 2024-11-14T19:35:31,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731612931976, max timestamp=1731612931993Num uniques 1018; 2024-11-14T19:35:31,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T19:35:31,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T19:35:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612991982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:31,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731612931976, max timestamp=1731612931993Num uniques 1018; 2024-11-14T19:35:31,999 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T19:35:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:31,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612991983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T19:35:32,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:32,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T19:35:32,021 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#144 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:32,022 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/6e4457458137444386966176e19f05b8 is 175, key is test_row_0/A:col10/1731612930396/Put/seqid=0 2024-11-14T19:35:32,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742002_1178 (size=31583) 2024-11-14T19:35:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612992065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612992066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612992070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612992104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612992104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T19:35:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612992184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612992184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612992184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,270 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:32,276 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114313202b547504142a4212a09b9e125c4_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114313202b547504142a4212a09b9e125c4_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:32,277 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f795a11b0d8f4aaeb51af0112b322870, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:32,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f795a11b0d8f4aaeb51af0112b322870 is 175, key is test_row_0/A:col10/1731612931777/Put/seqid=0 2024-11-14T19:35:32,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742003_1179 (size=65673) 2024-11-14T19:35:32,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f795a11b0d8f4aaeb51af0112b322870 2024-11-14T19:35:32,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T19:35:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T19:35:32,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612992324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612992324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/56b6a00c51a04856ab956f550f6d81de is 50, key is test_row_0/B:col10/1731612931777/Put/seqid=0 2024-11-14T19:35:32,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742004_1180 (size=12151) 2024-11-14T19:35:32,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/56b6a00c51a04856ab956f550f6d81de 2024-11-14T19:35:32,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612992394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612992394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612992394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b2a93ac12c874d72b5718e17301b63f5 is 50, key is test_row_0/C:col10/1731612931777/Put/seqid=0 2024-11-14T19:35:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742005_1181 (size=12151) 2024-11-14T19:35:32,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b2a93ac12c874d72b5718e17301b63f5 2024-11-14T19:35:32,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f795a11b0d8f4aaeb51af0112b322870 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f795a11b0d8f4aaeb51af0112b322870 2024-11-14T19:35:32,468 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/6e4457458137444386966176e19f05b8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/6e4457458137444386966176e19f05b8 2024-11-14T19:35:32,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T19:35:32,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:32,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:32,479 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into 6e4457458137444386966176e19f05b8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:32,480 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:32,480 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=12, startTime=1731612931501; duration=0sec 2024-11-14T19:35:32,480 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:32,480 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:32,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f795a11b0d8f4aaeb51af0112b322870, entries=350, sequenceid=210, filesize=64.1 K 2024-11-14T19:35:32,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/56b6a00c51a04856ab956f550f6d81de as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56b6a00c51a04856ab956f550f6d81de 2024-11-14T19:35:32,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56b6a00c51a04856ab956f550f6d81de, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T19:35:32,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b2a93ac12c874d72b5718e17301b63f5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b2a93ac12c874d72b5718e17301b63f5 2024-11-14T19:35:32,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b2a93ac12c874d72b5718e17301b63f5, entries=150, sequenceid=210, filesize=11.9 K 2024-11-14T19:35:32,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 965abd24f852442dbb51955455ef7db5 in 728ms, sequenceid=210, compaction requested=false 2024-11-14T19:35:32,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:32,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-14T19:35:32,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:32,626 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=134.18 KB heapSize=374.77 KB 2024-11-14T19:35:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:32,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:32,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149d8f8615506549818503dac0c941b111_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612932018/Put/seqid=0 2024-11-14T19:35:32,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:32,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742006_1182 (size=12404) 2024-11-14T19:35:32,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:32,693 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149d8f8615506549818503dac0c941b111_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149d8f8615506549818503dac0c941b111_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:32,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/44d6b38b7d8f41d3b1b6e34c700b2b82, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/44d6b38b7d8f41d3b1b6e34c700b2b82 is 175, key is test_row_0/A:col10/1731612932018/Put/seqid=0 2024-11-14T19:35:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612992691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612992690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612992703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612992703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612992714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742007_1183 (size=31205) 2024-11-14T19:35:32,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612992814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612992814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:32,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T19:35:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612993033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612993034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:33,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:33,134 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/44d6b38b7d8f41d3b1b6e34c700b2b82 2024-11-14T19:35:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f563206c09b542e9b82384289ae31aaa is 50, key is test_row_0/B:col10/1731612932018/Put/seqid=0 2024-11-14T19:35:33,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742008_1184 (size=12251) 2024-11-14T19:35:33,174 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f563206c09b542e9b82384289ae31aaa 2024-11-14T19:35:33,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/65ac58a16bdb4b53a523042fdac93d0f is 50, key is test_row_0/C:col10/1731612932018/Put/seqid=0 2024-11-14T19:35:33,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742009_1185 (size=12251) 2024-11-14T19:35:33,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612993223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:33,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612993223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:33,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612993233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:33,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612993353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:33,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612993353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:33,614 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/65ac58a16bdb4b53a523042fdac93d0f 2024-11-14T19:35:33,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/44d6b38b7d8f41d3b1b6e34c700b2b82 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/44d6b38b7d8f41d3b1b6e34c700b2b82 2024-11-14T19:35:33,628 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/44d6b38b7d8f41d3b1b6e34c700b2b82, entries=150, sequenceid=263, filesize=30.5 K 2024-11-14T19:35:33,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f563206c09b542e9b82384289ae31aaa as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f563206c09b542e9b82384289ae31aaa 2024-11-14T19:35:33,635 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f563206c09b542e9b82384289ae31aaa, entries=150, sequenceid=263, filesize=12.0 K 2024-11-14T19:35:33,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/65ac58a16bdb4b53a523042fdac93d0f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/65ac58a16bdb4b53a523042fdac93d0f 2024-11-14T19:35:33,643 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/65ac58a16bdb4b53a523042fdac93d0f, entries=150, sequenceid=263, filesize=12.0 K 2024-11-14T19:35:33,644 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~374.72 KB/383712, currentSize=53.67 KB/54960 for 965abd24f852442dbb51955455ef7db5 in 1018ms, sequenceid=263, compaction requested=true 2024-11-14T19:35:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:33,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-14T19:35:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-14T19:35:33,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.9560 sec 2024-11-14T19:35:33,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-14T19:35:33,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9470 sec 2024-11-14T19:35:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-14T19:35:33,843 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:33,844 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-14T19:35:33,846 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T19:35:33,847 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:33,847 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:33,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:35:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:33,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:33,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144b9c630e347c40cbbd6680f4f9c97824_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612932678/Put/seqid=0 2024-11-14T19:35:33,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742010_1186 (size=14994) 2024-11-14T19:35:33,950 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T19:35:33,956 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144b9c630e347c40cbbd6680f4f9c97824_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b9c630e347c40cbbd6680f4f9c97824_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:33,959 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ef07ae7be62842659a1983609b8f5b19, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:33,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ef07ae7be62842659a1983609b8f5b19 is 175, key is test_row_0/A:col10/1731612932678/Put/seqid=0 2024-11-14T19:35:33,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742011_1187 (size=39949) 2024-11-14T19:35:33,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ef07ae7be62842659a1983609b8f5b19 2024-11-14T19:35:34,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T19:35:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/dca998ac0686402fbba2e19179afa3fd is 50, key is test_row_0/B:col10/1731612932678/Put/seqid=0 2024-11-14T19:35:34,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612994007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612994009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742012_1188 (size=12301) 2024-11-14T19:35:34,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/dca998ac0686402fbba2e19179afa3fd 2024-11-14T19:35:34,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9d17aea4989e45f8b19bc38da4d55400 is 50, key is test_row_0/C:col10/1731612932678/Put/seqid=0 2024-11-14T19:35:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612994124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612994124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T19:35:34,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:34,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T19:35:34,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742013_1189 (size=12301) 2024-11-14T19:35:34,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612994244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612994253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,274 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:34,274 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612994244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,275 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T19:35:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612994343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612994344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T19:35:34,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T19:35:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:34,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9d17aea4989e45f8b19bc38da4d55400 2024-11-14T19:35:34,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ef07ae7be62842659a1983609b8f5b19 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ef07ae7be62842659a1983609b8f5b19 2024-11-14T19:35:34,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ef07ae7be62842659a1983609b8f5b19, entries=200, sequenceid=275, filesize=39.0 K 2024-11-14T19:35:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/dca998ac0686402fbba2e19179afa3fd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/dca998ac0686402fbba2e19179afa3fd 2024-11-14T19:35:34,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/dca998ac0686402fbba2e19179afa3fd, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T19:35:34,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9d17aea4989e45f8b19bc38da4d55400 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9d17aea4989e45f8b19bc38da4d55400 2024-11-14T19:35:34,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9d17aea4989e45f8b19bc38da4d55400, entries=150, sequenceid=275, filesize=12.0 K 2024-11-14T19:35:34,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 965abd24f852442dbb51955455ef7db5 in 772ms, sequenceid=275, compaction requested=true 2024-11-14T19:35:34,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:34,640 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:34,641 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:34,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:34,642 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 168410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:34,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:34,642 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:34,642 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,642 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/6e4457458137444386966176e19f05b8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f795a11b0d8f4aaeb51af0112b322870, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/44d6b38b7d8f41d3b1b6e34c700b2b82, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ef07ae7be62842659a1983609b8f5b19] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=164.5 K 2024-11-14T19:35:34,642 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,642 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/6e4457458137444386966176e19f05b8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f795a11b0d8f4aaeb51af0112b322870, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/44d6b38b7d8f41d3b1b6e34c700b2b82, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ef07ae7be62842659a1983609b8f5b19] 2024-11-14T19:35:34,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:34,645 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e4457458137444386966176e19f05b8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731612930396 2024-11-14T19:35:34,646 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:34,646 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:34,646 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,646 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/2dba437973884212b22ae997b23d4f80, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56b6a00c51a04856ab956f550f6d81de, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f563206c09b542e9b82384289ae31aaa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/dca998ac0686402fbba2e19179afa3fd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=48.2 K 2024-11-14T19:35:34,646 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting f795a11b0d8f4aaeb51af0112b322870, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731612931664 2024-11-14T19:35:34,647 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dba437973884212b22ae997b23d4f80, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731612930396 2024-11-14T19:35:34,647 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d6b38b7d8f41d3b1b6e34c700b2b82, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731612932018 2024-11-14T19:35:34,647 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b6a00c51a04856ab956f550f6d81de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731612931664 2024-11-14T19:35:34,647 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef07ae7be62842659a1983609b8f5b19, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612932678 2024-11-14T19:35:34,648 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f563206c09b542e9b82384289ae31aaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731612932018 2024-11-14T19:35:34,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-14T19:35:34,648 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting dca998ac0686402fbba2e19179afa3fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612932678 2024-11-14T19:35:34,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,648 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:35:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:34,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:34,692 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:34,693 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/b326569218334faf9768125731957436 is 50, key is test_row_0/B:col10/1731612932678/Put/seqid=0 2024-11-14T19:35:34,702 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:34,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148533a0fd69554c71879064d286b8ad1e_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612933969/Put/seqid=0 2024-11-14T19:35:34,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612994722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612994724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,746 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411144e8c16836ed34dca89b479686cdf1d19_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:34,748 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411144e8c16836ed34dca89b479686cdf1d19_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:34,749 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144e8c16836ed34dca89b479686cdf1d19_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:34,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742014_1190 (size=12915) 2024-11-14T19:35:34,783 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/b326569218334faf9768125731957436 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b326569218334faf9768125731957436 2024-11-14T19:35:34,792 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into b326569218334faf9768125731957436(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:34,792 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:34,792 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=12, startTime=1731612934641; duration=0sec 2024-11-14T19:35:34,792 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:34,792 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:34,792 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:35:34,794 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:35:34,794 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:34,794 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:34,794 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/89e296bbfb73423d95760f9eb3a043f2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b2a93ac12c874d72b5718e17301b63f5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/65ac58a16bdb4b53a523042fdac93d0f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9d17aea4989e45f8b19bc38da4d55400] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=48.2 K 2024-11-14T19:35:34,796 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e296bbfb73423d95760f9eb3a043f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731612930396 2024-11-14T19:35:34,796 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b2a93ac12c874d72b5718e17301b63f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731612931664 2024-11-14T19:35:34,796 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ac58a16bdb4b53a523042fdac93d0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731612932018 2024-11-14T19:35:34,797 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d17aea4989e45f8b19bc38da4d55400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612932678 2024-11-14T19:35:34,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742015_1191 (size=12454) 2024-11-14T19:35:34,813 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:34,814 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a3d4e2256c41409dac77666420a1090e is 50, key is test_row_0/C:col10/1731612932678/Put/seqid=0 2024-11-14T19:35:34,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742016_1192 (size=4469) 2024-11-14T19:35:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612994833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612994833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:34,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:34,846 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#157 average throughput is 0.17 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:34,847 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/48d6dd578cc444a1a8ef4f5ab3097425 is 175, key is test_row_0/A:col10/1731612932678/Put/seqid=0 2024-11-14T19:35:34,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742017_1193 (size=12915) 2024-11-14T19:35:34,878 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/a3d4e2256c41409dac77666420a1090e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a3d4e2256c41409dac77666420a1090e 2024-11-14T19:35:34,892 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into a3d4e2256c41409dac77666420a1090e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:34,892 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:34,892 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=12, startTime=1731612934642; duration=0sec 2024-11-14T19:35:34,892 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:34,892 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:34,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742018_1194 (size=31869) 2024-11-14T19:35:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T19:35:35,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612995043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:35,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:35,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612995043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:35,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:35,083 INFO [master/867b237d0fa7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T19:35:35,083 INFO [master/867b237d0fa7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T19:35:35,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:35,217 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148533a0fd69554c71879064d286b8ad1e_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148533a0fd69554c71879064d286b8ad1e_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:35,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/b00ec7d470a242a2b3998667b7cdbb48, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/b00ec7d470a242a2b3998667b7cdbb48 is 175, key is test_row_0/A:col10/1731612933969/Put/seqid=0 2024-11-14T19:35:35,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742019_1195 (size=31255) 2024-11-14T19:35:35,320 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/48d6dd578cc444a1a8ef4f5ab3097425 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/48d6dd578cc444a1a8ef4f5ab3097425 2024-11-14T19:35:35,327 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into 48d6dd578cc444a1a8ef4f5ab3097425(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:35,327 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:35,327 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=12, startTime=1731612934640; duration=0sec 2024-11-14T19:35:35,328 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:35,328 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:35,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612995353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:35,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:35,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612995353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:35,671 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/b00ec7d470a242a2b3998667b7cdbb48 2024-11-14T19:35:35,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/935d687026da41579551169141b1b986 is 50, key is test_row_0/B:col10/1731612933969/Put/seqid=0 2024-11-14T19:35:35,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742020_1196 (size=12301) 2024-11-14T19:35:35,709 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/935d687026da41579551169141b1b986 2024-11-14T19:35:35,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/7636995d7dbb41e49e430e8b22bfd7e4 is 50, key is test_row_0/C:col10/1731612933969/Put/seqid=0 2024-11-14T19:35:35,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742021_1197 (size=12301) 2024-11-14T19:35:35,751 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/7636995d7dbb41e49e430e8b22bfd7e4 2024-11-14T19:35:35,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/b00ec7d470a242a2b3998667b7cdbb48 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/b00ec7d470a242a2b3998667b7cdbb48 2024-11-14T19:35:35,770 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/b00ec7d470a242a2b3998667b7cdbb48, entries=150, sequenceid=295, filesize=30.5 K 2024-11-14T19:35:35,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/935d687026da41579551169141b1b986 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/935d687026da41579551169141b1b986 2024-11-14T19:35:35,780 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/935d687026da41579551169141b1b986, entries=150, sequenceid=295, filesize=12.0 K 2024-11-14T19:35:35,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/7636995d7dbb41e49e430e8b22bfd7e4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/7636995d7dbb41e49e430e8b22bfd7e4 2024-11-14T19:35:35,791 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/7636995d7dbb41e49e430e8b22bfd7e4, entries=150, sequenceid=295, filesize=12.0 K 2024-11-14T19:35:35,792 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 965abd24f852442dbb51955455ef7db5 in 1144ms, sequenceid=295, compaction requested=false 2024-11-14T19:35:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:35,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-14T19:35:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-14T19:35:35,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-14T19:35:35,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9470 sec 2024-11-14T19:35:35,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.9530 sec 2024-11-14T19:35:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:35,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:35:35,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:35,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:35,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:35,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:35,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:35,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:35,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111410f0fd6d02c54f97b598992608edfc70_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612934719/Put/seqid=0 2024-11-14T19:35:35,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742022_1198 (size=14994) 2024-11-14T19:35:35,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:35,918 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111410f0fd6d02c54f97b598992608edfc70_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410f0fd6d02c54f97b598992608edfc70_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:35,919 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/250abffce9ca431f93e33fb0f386932e, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:35,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/250abffce9ca431f93e33fb0f386932e is 175, key is test_row_0/A:col10/1731612934719/Put/seqid=0 2024-11-14T19:35:35,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612995945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:35,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612995946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:35,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742023_1199 (size=39949) 2024-11-14T19:35:35,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/250abffce9ca431f93e33fb0f386932e 2024-11-14T19:35:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:35,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:35,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/56903b67eb6e48338bbaecbcac3cc52b is 50, key is test_row_0/B:col10/1731612934719/Put/seqid=0 2024-11-14T19:35:35,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-14T19:35:35,994 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:35,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-14T19:35:35,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T19:35:35,997 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:35,998 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:35,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:36,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742024_1200 (size=12301) 2024-11-14T19:35:36,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612996063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612996073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T19:35:36,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T19:35:36,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:36,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731612996284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731612996284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,286 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:36,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612996284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,287 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731612996284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612996285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,288 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T19:35:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T19:35:36,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/56903b67eb6e48338bbaecbcac3cc52b 2024-11-14T19:35:36,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/90beaf7ad59d46af9cae8360caf11ada is 50, key is test_row_0/C:col10/1731612934719/Put/seqid=0 2024-11-14T19:35:36,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742025_1201 (size=12301) 2024-11-14T19:35:36,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T19:35:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612996593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612996593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:36,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T19:35:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T19:35:36,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T19:35:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:36,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/90beaf7ad59d46af9cae8360caf11ada 2024-11-14T19:35:36,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/250abffce9ca431f93e33fb0f386932e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/250abffce9ca431f93e33fb0f386932e 2024-11-14T19:35:36,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/250abffce9ca431f93e33fb0f386932e, entries=200, sequenceid=311, filesize=39.0 K 2024-11-14T19:35:36,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/56903b67eb6e48338bbaecbcac3cc52b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56903b67eb6e48338bbaecbcac3cc52b 2024-11-14T19:35:36,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56903b67eb6e48338bbaecbcac3cc52b, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T19:35:36,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/90beaf7ad59d46af9cae8360caf11ada as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/90beaf7ad59d46af9cae8360caf11ada 2024-11-14T19:35:36,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/90beaf7ad59d46af9cae8360caf11ada, entries=150, sequenceid=311, filesize=12.0 K 2024-11-14T19:35:36,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 965abd24f852442dbb51955455ef7db5 in 1001ms, sequenceid=311, compaction requested=true 2024-11-14T19:35:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:36,870 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:36,870 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:36,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:36,871 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:36,871 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:36,871 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,872 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/48d6dd578cc444a1a8ef4f5ab3097425, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/b00ec7d470a242a2b3998667b7cdbb48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/250abffce9ca431f93e33fb0f386932e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=100.7 K 2024-11-14T19:35:36,872 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,872 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/48d6dd578cc444a1a8ef4f5ab3097425, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/b00ec7d470a242a2b3998667b7cdbb48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/250abffce9ca431f93e33fb0f386932e] 2024-11-14T19:35:36,872 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:36,872 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48d6dd578cc444a1a8ef4f5ab3097425, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612932678 2024-11-14T19:35:36,872 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:36,873 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,873 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b326569218334faf9768125731957436, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/935d687026da41579551169141b1b986, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56903b67eb6e48338bbaecbcac3cc52b] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=36.6 K 2024-11-14T19:35:36,873 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00ec7d470a242a2b3998667b7cdbb48, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731612933969 2024-11-14T19:35:36,873 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b326569218334faf9768125731957436, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612932678 2024-11-14T19:35:36,874 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 250abffce9ca431f93e33fb0f386932e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731612934708 2024-11-14T19:35:36,875 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 935d687026da41579551169141b1b986, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731612933969 2024-11-14T19:35:36,875 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 56903b67eb6e48338bbaecbcac3cc52b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731612934719 2024-11-14T19:35:36,882 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:36,888 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:36,889 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f6698eb4d29a459e9926c92da6f3e4e6 is 50, key is test_row_0/B:col10/1731612934719/Put/seqid=0 2024-11-14T19:35:36,895 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411144da1e7baf14d44e78335cdec12fb94f9_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:36,897 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411144da1e7baf14d44e78335cdec12fb94f9_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:36,898 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144da1e7baf14d44e78335cdec12fb94f9_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:36,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-14T19:35:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,918 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:35:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:36,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:36,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742026_1202 (size=13017) 2024-11-14T19:35:36,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742027_1203 (size=4469) 2024-11-14T19:35:36,931 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/f6698eb4d29a459e9926c92da6f3e4e6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f6698eb4d29a459e9926c92da6f3e4e6 2024-11-14T19:35:36,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114afbb892fe3e243f2ad76b5ac4038e5b7_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612935943/Put/seqid=0 2024-11-14T19:35:36,941 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into f6698eb4d29a459e9926c92da6f3e4e6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:36,941 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:36,942 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=13, startTime=1731612936870; duration=0sec 2024-11-14T19:35:36,942 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:36,942 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:36,942 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:36,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742028_1204 (size=12454) 2024-11-14T19:35:36,944 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:36,944 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:36,944 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:36,944 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a3d4e2256c41409dac77666420a1090e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/7636995d7dbb41e49e430e8b22bfd7e4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/90beaf7ad59d46af9cae8360caf11ada] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=36.6 K 2024-11-14T19:35:36,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:36,944 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a3d4e2256c41409dac77666420a1090e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731612932678 2024-11-14T19:35:36,945 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 7636995d7dbb41e49e430e8b22bfd7e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731612933969 2024-11-14T19:35:36,945 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 90beaf7ad59d46af9cae8360caf11ada, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731612934719 2024-11-14T19:35:36,948 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114afbb892fe3e243f2ad76b5ac4038e5b7_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114afbb892fe3e243f2ad76b5ac4038e5b7_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:36,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5619284827dd40488a921be7c04c30bd, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:36,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5619284827dd40488a921be7c04c30bd is 175, key is test_row_0/A:col10/1731612935943/Put/seqid=0 2024-11-14T19:35:36,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742029_1205 (size=31255) 2024-11-14T19:35:36,961 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:36,962 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/ef55c1ac011546629784af8483e8e016 is 50, key is test_row_0/C:col10/1731612934719/Put/seqid=0 2024-11-14T19:35:36,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742030_1206 (size=13017) 2024-11-14T19:35:37,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T19:35:37,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612997149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612997150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612997262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612997263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,333 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#165 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:37,333 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/19db00c3ff4b45da9785c647979d771a is 175, key is test_row_0/A:col10/1731612934719/Put/seqid=0 2024-11-14T19:35:37,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742031_1207 (size=31971) 2024-11-14T19:35:37,343 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/19db00c3ff4b45da9785c647979d771a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/19db00c3ff4b45da9785c647979d771a 2024-11-14T19:35:37,348 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into 19db00c3ff4b45da9785c647979d771a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:37,348 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:37,348 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=13, startTime=1731612936870; duration=0sec 2024-11-14T19:35:37,349 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:37,349 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:37,360 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5619284827dd40488a921be7c04c30bd 2024-11-14T19:35:37,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/36303d081bd74983bedc732791e09093 is 50, key is test_row_0/B:col10/1731612935943/Put/seqid=0 2024-11-14T19:35:37,383 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/ef55c1ac011546629784af8483e8e016 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/ef55c1ac011546629784af8483e8e016 2024-11-14T19:35:37,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742032_1208 (size=12301) 2024-11-14T19:35:37,387 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/36303d081bd74983bedc732791e09093 2024-11-14T19:35:37,391 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into ef55c1ac011546629784af8483e8e016(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:37,391 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:37,391 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=13, startTime=1731612936870; duration=0sec 2024-11-14T19:35:37,391 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:37,391 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:37,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/45b6b97c024e42329b1c864f09f89ac1 is 50, key is test_row_0/C:col10/1731612935943/Put/seqid=0 2024-11-14T19:35:37,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742033_1209 (size=12301) 2024-11-14T19:35:37,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612997473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612997473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,492 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612997783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,803 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/45b6b97c024e42329b1c864f09f89ac1 2024-11-14T19:35:37,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612997803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:37,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:37,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/5619284827dd40488a921be7c04c30bd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5619284827dd40488a921be7c04c30bd 2024-11-14T19:35:37,812 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5619284827dd40488a921be7c04c30bd, entries=150, sequenceid=331, filesize=30.5 K 2024-11-14T19:35:37,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/36303d081bd74983bedc732791e09093 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/36303d081bd74983bedc732791e09093 2024-11-14T19:35:37,817 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/36303d081bd74983bedc732791e09093, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T19:35:37,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/45b6b97c024e42329b1c864f09f89ac1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/45b6b97c024e42329b1c864f09f89ac1 2024-11-14T19:35:37,825 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/45b6b97c024e42329b1c864f09f89ac1, entries=150, sequenceid=331, filesize=12.0 K 2024-11-14T19:35:37,826 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 965abd24f852442dbb51955455ef7db5 in 908ms, sequenceid=331, compaction requested=false 2024-11-14T19:35:37,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:37,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:37,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-14T19:35:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-14T19:35:37,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-14T19:35:37,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8300 sec 2024-11-14T19:35:37,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8360 sec 2024-11-14T19:35:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-14T19:35:38,143 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:38,144 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-14T19:35:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T19:35:38,146 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:38,146 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:38,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:38,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T19:35:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:38,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:35:38,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:38,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:38,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:38,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:38,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111409263061f7ad443d87dd70a6e7c17e73_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612937118/Put/seqid=0 2024-11-14T19:35:38,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742034_1210 (size=14994) 2024-11-14T19:35:38,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612998340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:38,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612998343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,345 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612998443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:38,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612998453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T19:35:38,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612998653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:38,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612998663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:38,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:38,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111409263061f7ad443d87dd70a6e7c17e73_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409263061f7ad443d87dd70a6e7c17e73_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:38,741 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ee64f6e4d3974fc8b9ad2c9646ecb595, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:38,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ee64f6e4d3974fc8b9ad2c9646ecb595 is 175, key is test_row_0/A:col10/1731612937118/Put/seqid=0 2024-11-14T19:35:38,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742035_1211 (size=39949) 2024-11-14T19:35:38,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T19:35:38,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:38,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:38,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612998963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:38,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612998973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:38,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:39,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,152 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=347, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ee64f6e4d3974fc8b9ad2c9646ecb595 2024-11-14T19:35:39,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/6a38cd21d2e44f078db15ba18ad8ae58 is 50, key is test_row_0/B:col10/1731612937118/Put/seqid=0 2024-11-14T19:35:39,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742036_1212 (size=12301) 2024-11-14T19:35:39,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T19:35:39,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731612999473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:39,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731612999483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:39,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:39,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/6a38cd21d2e44f078db15ba18ad8ae58 2024-11-14T19:35:39,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9765c398a6104c0cab6366cdf5ed92dd is 50, key is test_row_0/C:col10/1731612937118/Put/seqid=0 2024-11-14T19:35:39,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742037_1213 (size=12301) 2024-11-14T19:35:39,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:39,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:39,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:39,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:39,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:39,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:39,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:40,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9765c398a6104c0cab6366cdf5ed92dd 2024-11-14T19:35:40,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/ee64f6e4d3974fc8b9ad2c9646ecb595 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ee64f6e4d3974fc8b9ad2c9646ecb595 2024-11-14T19:35:40,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ee64f6e4d3974fc8b9ad2c9646ecb595, entries=200, sequenceid=347, filesize=39.0 K 2024-11-14T19:35:40,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/6a38cd21d2e44f078db15ba18ad8ae58 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/6a38cd21d2e44f078db15ba18ad8ae58 2024-11-14T19:35:40,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/6a38cd21d2e44f078db15ba18ad8ae58, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T19:35:40,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/9765c398a6104c0cab6366cdf5ed92dd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9765c398a6104c0cab6366cdf5ed92dd 2024-11-14T19:35:40,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9765c398a6104c0cab6366cdf5ed92dd, entries=150, sequenceid=347, filesize=12.0 K 2024-11-14T19:35:40,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 965abd24f852442dbb51955455ef7db5 in 1738ms, sequenceid=347, compaction requested=true 2024-11-14T19:35:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:40,035 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:40,035 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:40,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:40,036 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:40,036 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:40,036 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:40,036 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:40,036 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:40,036 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:40,036 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f6698eb4d29a459e9926c92da6f3e4e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/36303d081bd74983bedc732791e09093, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/6a38cd21d2e44f078db15ba18ad8ae58] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=36.7 K 2024-11-14T19:35:40,036 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/19db00c3ff4b45da9785c647979d771a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5619284827dd40488a921be7c04c30bd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ee64f6e4d3974fc8b9ad2c9646ecb595] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=100.8 K 2024-11-14T19:35:40,036 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:40,036 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/19db00c3ff4b45da9785c647979d771a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5619284827dd40488a921be7c04c30bd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ee64f6e4d3974fc8b9ad2c9646ecb595] 2024-11-14T19:35:40,036 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f6698eb4d29a459e9926c92da6f3e4e6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731612934719 2024-11-14T19:35:40,037 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19db00c3ff4b45da9785c647979d771a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731612934719 2024-11-14T19:35:40,037 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 36303d081bd74983bedc732791e09093, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731612935921 2024-11-14T19:35:40,037 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5619284827dd40488a921be7c04c30bd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731612935921 2024-11-14T19:35:40,037 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a38cd21d2e44f078db15ba18ad8ae58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731612937118 2024-11-14T19:35:40,037 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee64f6e4d3974fc8b9ad2c9646ecb595, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731612937113 2024-11-14T19:35:40,042 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:40,044 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:40,044 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/b07d8c3f3fd64b1ab176823fae27def6 is 50, key is test_row_0/B:col10/1731612937118/Put/seqid=0 2024-11-14T19:35:40,047 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b2ce06f59b0e45adba51c4037ea28020_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:40,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742038_1214 (size=13119) 2024-11-14T19:35:40,050 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b2ce06f59b0e45adba51c4037ea28020_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:40,050 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b2ce06f59b0e45adba51c4037ea28020_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:40,057 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/b07d8c3f3fd64b1ab176823fae27def6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b07d8c3f3fd64b1ab176823fae27def6 2024-11-14T19:35:40,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742039_1215 (size=4469) 2024-11-14T19:35:40,063 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into b07d8c3f3fd64b1ab176823fae27def6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:40,063 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:40,063 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=13, startTime=1731612940035; duration=0sec 2024-11-14T19:35:40,063 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:40,063 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:40,063 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:40,064 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:40,064 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:40,064 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:40,065 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/ef55c1ac011546629784af8483e8e016, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/45b6b97c024e42329b1c864f09f89ac1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9765c398a6104c0cab6366cdf5ed92dd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=36.7 K 2024-11-14T19:35:40,065 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ef55c1ac011546629784af8483e8e016, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731612934719 2024-11-14T19:35:40,066 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 45b6b97c024e42329b1c864f09f89ac1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731612935921 2024-11-14T19:35:40,066 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9765c398a6104c0cab6366cdf5ed92dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731612937118 2024-11-14T19:35:40,077 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:40,077 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/8e1fc9af69794509b58eb7b4e5cf1f06 is 50, key is test_row_0/C:col10/1731612937118/Put/seqid=0 2024-11-14T19:35:40,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742040_1216 (size=13119) 2024-11-14T19:35:40,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-14T19:35:40,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:40,140 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T19:35:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:40,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114025dbcba758343a69ee3ec42e6bdbf6d_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612938331/Put/seqid=0 2024-11-14T19:35:40,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742041_1217 (size=12454) 2024-11-14T19:35:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T19:35:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:40,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. as already flushing 2024-11-14T19:35:40,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731613000323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731613000324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60448 deadline: 1731613000333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,339 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,349 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-11-14T19:35:40,349 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:40,349 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,349 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:40,354 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-11-14T19:35:40,355 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:40,355 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,355 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:40,355 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-14T19:35:40,355 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:40,355 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,355 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:40,357 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-14T19:35:40,357 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:40,357 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,357 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:40,357 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:40,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731613000434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731613000434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,460 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#174 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:40,461 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/306bf74379c74b369f3212df9e643eb0 is 175, key is test_row_0/A:col10/1731612937118/Put/seqid=0 2024-11-14T19:35:40,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742042_1218 (size=32073) 2024-11-14T19:35:40,487 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/8e1fc9af69794509b58eb7b4e5cf1f06 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8e1fc9af69794509b58eb7b4e5cf1f06 2024-11-14T19:35:40,493 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into 8e1fc9af69794509b58eb7b4e5cf1f06(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:40,493 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:40,493 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=13, startTime=1731612940035; duration=0sec 2024-11-14T19:35:40,494 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:40,494 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:40,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731613000493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,494 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1731613000503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,504 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 965abd24f852442dbb51955455ef7db5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:40,561 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114025dbcba758343a69ee3ec42e6bdbf6d_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114025dbcba758343a69ee3ec42e6bdbf6d_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f2c62bc2025841e2b00fb84aea331bcd, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f2c62bc2025841e2b00fb84aea331bcd is 175, key is test_row_0/A:col10/1731612938331/Put/seqid=0 2024-11-14T19:35:40,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742043_1219 (size=31255) 2024-11-14T19:35:40,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731613000643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731613000643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,871 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/306bf74379c74b369f3212df9e643eb0 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/306bf74379c74b369f3212df9e643eb0 2024-11-14T19:35:40,878 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into 306bf74379c74b369f3212df9e643eb0(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:40,878 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:40,878 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=13, startTime=1731612940035; duration=0sec 2024-11-14T19:35:40,878 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:40,878 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731613000953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731613000953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:40,968 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f2c62bc2025841e2b00fb84aea331bcd 2024-11-14T19:35:40,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/78120fb6e8844e7980dac5526e38d736 is 50, key is test_row_0/B:col10/1731612938331/Put/seqid=0 2024-11-14T19:35:40,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742044_1220 (size=12301) 2024-11-14T19:35:41,385 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/78120fb6e8844e7980dac5526e38d736 2024-11-14T19:35:41,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/cbdbd9550d774b2b9d6d73b1ef3eddd0 is 50, key is test_row_0/C:col10/1731612938331/Put/seqid=0 2024-11-14T19:35:41,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742045_1221 (size=12301) 2024-11-14T19:35:41,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:41,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1731613001464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60518 deadline: 1731613001464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:41,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:41,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:35:41,804 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/cbdbd9550d774b2b9d6d73b1ef3eddd0 2024-11-14T19:35:41,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/f2c62bc2025841e2b00fb84aea331bcd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f2c62bc2025841e2b00fb84aea331bcd 2024-11-14T19:35:41,822 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f2c62bc2025841e2b00fb84aea331bcd, entries=150, sequenceid=367, filesize=30.5 K 2024-11-14T19:35:41,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/78120fb6e8844e7980dac5526e38d736 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/78120fb6e8844e7980dac5526e38d736 2024-11-14T19:35:41,829 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/78120fb6e8844e7980dac5526e38d736, entries=150, sequenceid=367, filesize=12.0 K 2024-11-14T19:35:41,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/cbdbd9550d774b2b9d6d73b1ef3eddd0 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/cbdbd9550d774b2b9d6d73b1ef3eddd0 2024-11-14T19:35:41,837 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/cbdbd9550d774b2b9d6d73b1ef3eddd0, entries=150, sequenceid=367, filesize=12.0 K 2024-11-14T19:35:41,838 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 965abd24f852442dbb51955455ef7db5 in 1699ms, sequenceid=367, compaction requested=false 2024-11-14T19:35:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:41,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-14T19:35:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-14T19:35:41,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-14T19:35:41,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6930 sec 2024-11-14T19:35:41,844 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 3.6990 sec 2024-11-14T19:35:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-14T19:35:42,314 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:42,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T19:35:42,474 INFO [Thread-598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-598. 2024-11-14T19:35:42,475 DEBUG [Thread-598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:42,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:42,475 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:42,475 INFO [Thread-602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-602. 2024-11-14T19:35:42,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:42,475 DEBUG [Thread-598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:42,475 DEBUG [Thread-602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:42,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:42,475 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:42,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:42,475 DEBUG [Thread-602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,475 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:42,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f23d2119cbf499b98080abe0acbb6e8_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_0/A:col10/1731612940322/Put/seqid=0 2024-11-14T19:35:42,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742046_1222 (size=12454) 2024-11-14T19:35:42,515 INFO [Thread-596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-596. 2024-11-14T19:35:42,515 DEBUG [Thread-596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:42,515 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,515 DEBUG [Thread-596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:42,534 INFO [Thread-604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-604. 2024-11-14T19:35:42,534 DEBUG [Thread-604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:42,534 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,534 DEBUG [Thread-604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:42,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:42,887 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:42,891 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143f23d2119cbf499b98080abe0acbb6e8_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f23d2119cbf499b98080abe0acbb6e8_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:42,892 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/286e3951ae5347b081505056432d0f07, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:42,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/286e3951ae5347b081505056432d0f07 is 175, key is test_row_0/A:col10/1731612940322/Put/seqid=0 2024-11-14T19:35:42,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742047_1223 (size=31255) 2024-11-14T19:35:43,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/286e3951ae5347b081505056432d0f07 2024-11-14T19:35:43,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/a6f8771edbd6406684b5880bd89c35f2 is 50, key is test_row_0/B:col10/1731612940322/Put/seqid=0 2024-11-14T19:35:43,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742048_1224 (size=12301) 2024-11-14T19:35:43,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/a6f8771edbd6406684b5880bd89c35f2 2024-11-14T19:35:43,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/eeda14965c73480e8740c055d6ea52ea is 50, key is test_row_0/C:col10/1731612940322/Put/seqid=0 2024-11-14T19:35:43,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742049_1225 (size=12301) 2024-11-14T19:35:44,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/eeda14965c73480e8740c055d6ea52ea 2024-11-14T19:35:44,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/286e3951ae5347b081505056432d0f07 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/286e3951ae5347b081505056432d0f07 2024-11-14T19:35:44,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/286e3951ae5347b081505056432d0f07, entries=150, sequenceid=384, filesize=30.5 K 2024-11-14T19:35:44,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/a6f8771edbd6406684b5880bd89c35f2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a6f8771edbd6406684b5880bd89c35f2 2024-11-14T19:35:44,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a6f8771edbd6406684b5880bd89c35f2, entries=150, sequenceid=384, filesize=12.0 K 2024-11-14T19:35:44,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/eeda14965c73480e8740c055d6ea52ea as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/eeda14965c73480e8740c055d6ea52ea 2024-11-14T19:35:44,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/eeda14965c73480e8740c055d6ea52ea, entries=150, sequenceid=384, filesize=12.0 K 2024-11-14T19:35:44,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 965abd24f852442dbb51955455ef7db5 in 1706ms, sequenceid=384, compaction requested=true 2024-11-14T19:35:44,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:44,181 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:44,181 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 965abd24f852442dbb51955455ef7db5:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:44,182 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:44,182 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/A is initiating minor compaction (all files) 2024-11-14T19:35:44,182 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:44,182 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/B is initiating minor compaction (all files) 2024-11-14T19:35:44,182 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/A in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:44,182 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/B in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:44,182 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/306bf74379c74b369f3212df9e643eb0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f2c62bc2025841e2b00fb84aea331bcd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/286e3951ae5347b081505056432d0f07] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=92.4 K 2024-11-14T19:35:44,182 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b07d8c3f3fd64b1ab176823fae27def6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/78120fb6e8844e7980dac5526e38d736, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a6f8771edbd6406684b5880bd89c35f2] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=36.8 K 2024-11-14T19:35:44,182 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:44,182 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/306bf74379c74b369f3212df9e643eb0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f2c62bc2025841e2b00fb84aea331bcd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/286e3951ae5347b081505056432d0f07] 2024-11-14T19:35:44,183 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b07d8c3f3fd64b1ab176823fae27def6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731612937118 2024-11-14T19:35:44,183 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 306bf74379c74b369f3212df9e643eb0, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731612937118 2024-11-14T19:35:44,183 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 78120fb6e8844e7980dac5526e38d736, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731612938331 2024-11-14T19:35:44,183 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c62bc2025841e2b00fb84aea331bcd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731612938331 2024-11-14T19:35:44,183 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a6f8771edbd6406684b5880bd89c35f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731612940322 2024-11-14T19:35:44,183 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 286e3951ae5347b081505056432d0f07, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731612940322 2024-11-14T19:35:44,195 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:44,199 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#B#compaction#183 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:44,199 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/d0772e99e9cb4be9a5f840fcac3393b8 is 50, key is test_row_0/B:col10/1731612940322/Put/seqid=0 2024-11-14T19:35:44,202 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111434da43dabae44136a01cdea4e1d92394_965abd24f852442dbb51955455ef7db5 store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742050_1226 (size=13221) 2024-11-14T19:35:44,234 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/d0772e99e9cb4be9a5f840fcac3393b8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/d0772e99e9cb4be9a5f840fcac3393b8 2024-11-14T19:35:44,239 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111434da43dabae44136a01cdea4e1d92394_965abd24f852442dbb51955455ef7db5, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:44,240 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111434da43dabae44136a01cdea4e1d92394_965abd24f852442dbb51955455ef7db5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:44,241 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/B of 965abd24f852442dbb51955455ef7db5 into d0772e99e9cb4be9a5f840fcac3393b8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:44,241 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:44,241 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/B, priority=13, startTime=1731612944181; duration=0sec 2024-11-14T19:35:44,241 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:44,241 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:B 2024-11-14T19:35:44,241 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:44,242 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:44,243 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 965abd24f852442dbb51955455ef7db5/C is initiating minor compaction (all files) 2024-11-14T19:35:44,243 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 965abd24f852442dbb51955455ef7db5/C in TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:44,243 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8e1fc9af69794509b58eb7b4e5cf1f06, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/cbdbd9550d774b2b9d6d73b1ef3eddd0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/eeda14965c73480e8740c055d6ea52ea] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp, totalSize=36.8 K 2024-11-14T19:35:44,243 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e1fc9af69794509b58eb7b4e5cf1f06, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731612937118 2024-11-14T19:35:44,244 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cbdbd9550d774b2b9d6d73b1ef3eddd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731612938331 2024-11-14T19:35:44,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742051_1227 (size=4469) 2024-11-14T19:35:44,245 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting eeda14965c73480e8740c055d6ea52ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731612940322 2024-11-14T19:35:44,246 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#A#compaction#184 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:44,247 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/a8afb326526b4911abaefa1ce9fe23d1 is 175, key is test_row_0/A:col10/1731612940322/Put/seqid=0 2024-11-14T19:35:44,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742052_1228 (size=32175) 2024-11-14T19:35:44,269 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/a8afb326526b4911abaefa1ce9fe23d1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/a8afb326526b4911abaefa1ce9fe23d1 2024-11-14T19:35:44,270 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 965abd24f852442dbb51955455ef7db5#C#compaction#185 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:44,271 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/c45d7f8d37f5473f86c4162381c7a25e is 50, key is test_row_0/C:col10/1731612940322/Put/seqid=0 2024-11-14T19:35:44,279 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/A of 965abd24f852442dbb51955455ef7db5 into a8afb326526b4911abaefa1ce9fe23d1(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:44,279 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:44,279 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/A, priority=13, startTime=1731612944180; duration=0sec 2024-11-14T19:35:44,279 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:44,279 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:A 2024-11-14T19:35:44,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742053_1229 (size=13221) 2024-11-14T19:35:44,290 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/c45d7f8d37f5473f86c4162381c7a25e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/c45d7f8d37f5473f86c4162381c7a25e 2024-11-14T19:35:44,304 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 965abd24f852442dbb51955455ef7db5/C of 965abd24f852442dbb51955455ef7db5 into c45d7f8d37f5473f86c4162381c7a25e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:44,304 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 965abd24f852442dbb51955455ef7db5: 2024-11-14T19:35:44,304 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., storeName=965abd24f852442dbb51955455ef7db5/C, priority=13, startTime=1731612944181; duration=0sec 2024-11-14T19:35:44,304 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:44,304 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 965abd24f852442dbb51955455ef7db5:C 2024-11-14T19:35:46,860 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T19:35:50,445 INFO [Thread-600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-600. 2024-11-14T19:35:50,445 DEBUG [Thread-600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:35:50,445 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:50,445 DEBUG [Thread-600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:50,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2038 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1973 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 909 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2724 rows 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 930 2024-11-14T19:35:50,446 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2787 rows 2024-11-14T19:35:50,446 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:35:50,446 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:35:50,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:50,446 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:50,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:35:50,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T19:35:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T19:35:50,450 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612950449"}]},"ts":"1731612950449"} 2024-11-14T19:35:50,451 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T19:35:50,451 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T19:35:50,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:35:50,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, UNASSIGN}] 2024-11-14T19:35:50,456 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, UNASSIGN 2024-11-14T19:35:50,457 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:50,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, UNASSIGN because future has completed 2024-11-14T19:35:50,459 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:35:50,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:35:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T19:35:50,611 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:50,611 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 965abd24f852442dbb51955455ef7db5, disabling compactions & flushes 2024-11-14T19:35:50,612 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. after waiting 0 ms 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:50,612 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 965abd24f852442dbb51955455ef7db5 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=A 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=B 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 965abd24f852442dbb51955455ef7db5, store=C 2024-11-14T19:35:50,612 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:50,641 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143bdb1755a14d45838899064494c3faa2_965abd24f852442dbb51955455ef7db5 is 50, key is test_row_1/A:col10/1731612950444/Put/seqid=0 2024-11-14T19:35:50,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742054_1230 (size=9914) 2024-11-14T19:35:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T19:35:51,045 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:51,050 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143bdb1755a14d45838899064494c3faa2_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143bdb1755a14d45838899064494c3faa2_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:51,051 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/683374359f1f4ce8940bf172a6757af7, store: [table=TestAcidGuarantees family=A region=965abd24f852442dbb51955455ef7db5] 2024-11-14T19:35:51,052 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/683374359f1f4ce8940bf172a6757af7 is 175, key is test_row_1/A:col10/1731612950444/Put/seqid=0 2024-11-14T19:35:51,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742055_1231 (size=22561) 2024-11-14T19:35:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T19:35:51,456 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=393, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/683374359f1f4ce8940bf172a6757af7 2024-11-14T19:35:51,464 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/b72925a33c7b411680a63f502af01971 is 50, key is test_row_1/B:col10/1731612950444/Put/seqid=0 2024-11-14T19:35:51,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742056_1232 (size=9857) 2024-11-14T19:35:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T19:35:51,869 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/b72925a33c7b411680a63f502af01971 2024-11-14T19:35:51,876 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b7ed0de5772a4928b8759c6f3720ff1a is 50, key is test_row_1/C:col10/1731612950444/Put/seqid=0 2024-11-14T19:35:51,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742057_1233 (size=9857) 2024-11-14T19:35:52,294 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b7ed0de5772a4928b8759c6f3720ff1a 2024-11-14T19:35:52,300 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/A/683374359f1f4ce8940bf172a6757af7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/683374359f1f4ce8940bf172a6757af7 2024-11-14T19:35:52,304 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/683374359f1f4ce8940bf172a6757af7, entries=100, sequenceid=393, filesize=22.0 K 2024-11-14T19:35:52,305 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/B/b72925a33c7b411680a63f502af01971 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b72925a33c7b411680a63f502af01971 2024-11-14T19:35:52,310 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b72925a33c7b411680a63f502af01971, entries=100, sequenceid=393, filesize=9.6 K 2024-11-14T19:35:52,311 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/.tmp/C/b7ed0de5772a4928b8759c6f3720ff1a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7ed0de5772a4928b8759c6f3720ff1a 2024-11-14T19:35:52,315 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7ed0de5772a4928b8759c6f3720ff1a, entries=100, sequenceid=393, filesize=9.6 K 2024-11-14T19:35:52,316 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 965abd24f852442dbb51955455ef7db5 in 1704ms, sequenceid=393, compaction requested=false 2024-11-14T19:35:52,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/690e2996f3674489a4812ccb0a8e5df5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/7e654dbfaf5e48e883edfa384d14e286, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/fa2b1c85fc8944d0bac2164fc6092010, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/271588a7aaf74c0e94e301b244411c5f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/d34d6f13128445b7b08f890d4fef2f48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/69aec367d19e450bbffd86cbc3443e4f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f000a6e0baff4bb487c153ffa8daa4c6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/08ab3c18d18b4f2787abebf06ca00afe, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5325de52c9814776b4a2b616b9aca6d4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5db3dce413f14560ad45d4918a59a6e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5bf40b7408334d05bfb4fdb53bfc72e4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5098ed22dd194f7d91c9d4d6a7d35101, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/3d5213e343d741ab821e793d97bd25b0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/e021770de2174e09bcb85ef33d752e99, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/6e4457458137444386966176e19f05b8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f795a11b0d8f4aaeb51af0112b322870, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/44d6b38b7d8f41d3b1b6e34c700b2b82, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ef07ae7be62842659a1983609b8f5b19, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/48d6dd578cc444a1a8ef4f5ab3097425, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/b00ec7d470a242a2b3998667b7cdbb48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/250abffce9ca431f93e33fb0f386932e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/19db00c3ff4b45da9785c647979d771a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5619284827dd40488a921be7c04c30bd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ee64f6e4d3974fc8b9ad2c9646ecb595, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/306bf74379c74b369f3212df9e643eb0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f2c62bc2025841e2b00fb84aea331bcd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/286e3951ae5347b081505056432d0f07] to archive 2024-11-14T19:35:52,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:35:52,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/690e2996f3674489a4812ccb0a8e5df5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/690e2996f3674489a4812ccb0a8e5df5 2024-11-14T19:35:52,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/7e654dbfaf5e48e883edfa384d14e286 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/7e654dbfaf5e48e883edfa384d14e286 2024-11-14T19:35:52,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/fa2b1c85fc8944d0bac2164fc6092010 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/fa2b1c85fc8944d0bac2164fc6092010 2024-11-14T19:35:52,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/271588a7aaf74c0e94e301b244411c5f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/271588a7aaf74c0e94e301b244411c5f 2024-11-14T19:35:52,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/d34d6f13128445b7b08f890d4fef2f48 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/d34d6f13128445b7b08f890d4fef2f48 2024-11-14T19:35:52,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/69aec367d19e450bbffd86cbc3443e4f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/69aec367d19e450bbffd86cbc3443e4f 2024-11-14T19:35:52,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f000a6e0baff4bb487c153ffa8daa4c6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f000a6e0baff4bb487c153ffa8daa4c6 2024-11-14T19:35:52,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/08ab3c18d18b4f2787abebf06ca00afe to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/08ab3c18d18b4f2787abebf06ca00afe 2024-11-14T19:35:52,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5325de52c9814776b4a2b616b9aca6d4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5325de52c9814776b4a2b616b9aca6d4 2024-11-14T19:35:52,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5db3dce413f14560ad45d4918a59a6e6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5db3dce413f14560ad45d4918a59a6e6 2024-11-14T19:35:52,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5bf40b7408334d05bfb4fdb53bfc72e4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5bf40b7408334d05bfb4fdb53bfc72e4 2024-11-14T19:35:52,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5098ed22dd194f7d91c9d4d6a7d35101 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5098ed22dd194f7d91c9d4d6a7d35101 2024-11-14T19:35:52,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/3d5213e343d741ab821e793d97bd25b0 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/3d5213e343d741ab821e793d97bd25b0 2024-11-14T19:35:52,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/e021770de2174e09bcb85ef33d752e99 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/e021770de2174e09bcb85ef33d752e99 2024-11-14T19:35:52,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/6e4457458137444386966176e19f05b8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/6e4457458137444386966176e19f05b8 2024-11-14T19:35:52,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f795a11b0d8f4aaeb51af0112b322870 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f795a11b0d8f4aaeb51af0112b322870 2024-11-14T19:35:52,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/44d6b38b7d8f41d3b1b6e34c700b2b82 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/44d6b38b7d8f41d3b1b6e34c700b2b82 2024-11-14T19:35:52,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ef07ae7be62842659a1983609b8f5b19 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ef07ae7be62842659a1983609b8f5b19 2024-11-14T19:35:52,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/48d6dd578cc444a1a8ef4f5ab3097425 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/48d6dd578cc444a1a8ef4f5ab3097425 2024-11-14T19:35:52,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/b00ec7d470a242a2b3998667b7cdbb48 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/b00ec7d470a242a2b3998667b7cdbb48 2024-11-14T19:35:52,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/250abffce9ca431f93e33fb0f386932e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/250abffce9ca431f93e33fb0f386932e 2024-11-14T19:35:52,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/19db00c3ff4b45da9785c647979d771a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/19db00c3ff4b45da9785c647979d771a 2024-11-14T19:35:52,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5619284827dd40488a921be7c04c30bd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/5619284827dd40488a921be7c04c30bd 2024-11-14T19:35:52,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ee64f6e4d3974fc8b9ad2c9646ecb595 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/ee64f6e4d3974fc8b9ad2c9646ecb595 2024-11-14T19:35:52,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/306bf74379c74b369f3212df9e643eb0 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/306bf74379c74b369f3212df9e643eb0 2024-11-14T19:35:52,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f2c62bc2025841e2b00fb84aea331bcd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/f2c62bc2025841e2b00fb84aea331bcd 2024-11-14T19:35:52,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/286e3951ae5347b081505056432d0f07 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/286e3951ae5347b081505056432d0f07 2024-11-14T19:35:52,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/480f92d0e2294bd9a6a2e6bb62b7a416, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/bff7948da738494395d535893c9170c2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/73e13923d6254283b5ad07b434b37304, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/65d46f543cba4514bb4968e844c3ce8d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/1bcdcb2ae8ff4771be45bf568a2475d9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/4f0f7e81c97e4e6a812480cf481feaeb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f107aeb88e8344c989c684431cb30125, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/c6f2d711f86946eebe2cf3720edee464, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a5c5a32df7a04add8a83b6cf53bd44ca, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9f756315bd324eebbd11a51cc6a4773f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/523b9fccc41846e98d2befa1fa991ec7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9d75c5e01fed4a7db0ed3c62e57ed452, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/57d93988cc6043ed92825cd791db5c73, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/2dba437973884212b22ae997b23d4f80, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/cf81c55ff6004b74a7cb97578944f7eb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56b6a00c51a04856ab956f550f6d81de, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f563206c09b542e9b82384289ae31aaa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b326569218334faf9768125731957436, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/dca998ac0686402fbba2e19179afa3fd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/935d687026da41579551169141b1b986, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f6698eb4d29a459e9926c92da6f3e4e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56903b67eb6e48338bbaecbcac3cc52b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/36303d081bd74983bedc732791e09093, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b07d8c3f3fd64b1ab176823fae27def6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/6a38cd21d2e44f078db15ba18ad8ae58, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/78120fb6e8844e7980dac5526e38d736, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a6f8771edbd6406684b5880bd89c35f2] to archive 2024-11-14T19:35:52,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:35:52,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/480f92d0e2294bd9a6a2e6bb62b7a416 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/480f92d0e2294bd9a6a2e6bb62b7a416 2024-11-14T19:35:52,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/bff7948da738494395d535893c9170c2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/bff7948da738494395d535893c9170c2 2024-11-14T19:35:52,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/73e13923d6254283b5ad07b434b37304 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/73e13923d6254283b5ad07b434b37304 2024-11-14T19:35:52,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/65d46f543cba4514bb4968e844c3ce8d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/65d46f543cba4514bb4968e844c3ce8d 2024-11-14T19:35:52,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/1bcdcb2ae8ff4771be45bf568a2475d9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/1bcdcb2ae8ff4771be45bf568a2475d9 2024-11-14T19:35:52,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/4f0f7e81c97e4e6a812480cf481feaeb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/4f0f7e81c97e4e6a812480cf481feaeb 2024-11-14T19:35:52,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f107aeb88e8344c989c684431cb30125 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f107aeb88e8344c989c684431cb30125 2024-11-14T19:35:52,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/c6f2d711f86946eebe2cf3720edee464 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/c6f2d711f86946eebe2cf3720edee464 2024-11-14T19:35:52,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a5c5a32df7a04add8a83b6cf53bd44ca to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a5c5a32df7a04add8a83b6cf53bd44ca 2024-11-14T19:35:52,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9f756315bd324eebbd11a51cc6a4773f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9f756315bd324eebbd11a51cc6a4773f 2024-11-14T19:35:52,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/523b9fccc41846e98d2befa1fa991ec7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/523b9fccc41846e98d2befa1fa991ec7 2024-11-14T19:35:52,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9d75c5e01fed4a7db0ed3c62e57ed452 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/9d75c5e01fed4a7db0ed3c62e57ed452 2024-11-14T19:35:52,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/57d93988cc6043ed92825cd791db5c73 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/57d93988cc6043ed92825cd791db5c73 2024-11-14T19:35:52,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/2dba437973884212b22ae997b23d4f80 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/2dba437973884212b22ae997b23d4f80 2024-11-14T19:35:52,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/cf81c55ff6004b74a7cb97578944f7eb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/cf81c55ff6004b74a7cb97578944f7eb 2024-11-14T19:35:52,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56b6a00c51a04856ab956f550f6d81de to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56b6a00c51a04856ab956f550f6d81de 2024-11-14T19:35:52,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f563206c09b542e9b82384289ae31aaa to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f563206c09b542e9b82384289ae31aaa 2024-11-14T19:35:52,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b326569218334faf9768125731957436 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b326569218334faf9768125731957436 2024-11-14T19:35:52,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/dca998ac0686402fbba2e19179afa3fd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/dca998ac0686402fbba2e19179afa3fd 2024-11-14T19:35:52,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/935d687026da41579551169141b1b986 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/935d687026da41579551169141b1b986 2024-11-14T19:35:52,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f6698eb4d29a459e9926c92da6f3e4e6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/f6698eb4d29a459e9926c92da6f3e4e6 2024-11-14T19:35:52,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56903b67eb6e48338bbaecbcac3cc52b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/56903b67eb6e48338bbaecbcac3cc52b 2024-11-14T19:35:52,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/36303d081bd74983bedc732791e09093 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/36303d081bd74983bedc732791e09093 2024-11-14T19:35:52,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b07d8c3f3fd64b1ab176823fae27def6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b07d8c3f3fd64b1ab176823fae27def6 2024-11-14T19:35:52,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/6a38cd21d2e44f078db15ba18ad8ae58 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/6a38cd21d2e44f078db15ba18ad8ae58 2024-11-14T19:35:52,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/78120fb6e8844e7980dac5526e38d736 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/78120fb6e8844e7980dac5526e38d736 2024-11-14T19:35:52,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a6f8771edbd6406684b5880bd89c35f2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/a6f8771edbd6406684b5880bd89c35f2 2024-11-14T19:35:52,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a7d0d76bf4a24d09b908b6d3e845ce74, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/22be4069e35747f4ab52a111a1c3e884, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/e96b0406fc704087b141ba500a1dcbfa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/37c2b2a1fc1345bd8d15104f431801e3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/4afe457a5d144acc8e228bd230ed25d5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b49f49b5395d418cb16bc8e29a982624, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7311b53f95b404ab4d1805d1193f843, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/199cf73a0cd84bc7b913ed737a10bd4b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8759b9faea804975b3ed3dccd60491f5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/f630c30195d3481986bb9a9f7e10e9ad, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/1e5aaffdbfe64858a649cc8dd91674a8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9db44299e1c2448cae23299789586960, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/5af4ace65d574eecba69faee30761bee, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/89e296bbfb73423d95760f9eb3a043f2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a2b04896158144b2840366d2d32d7ab5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b2a93ac12c874d72b5718e17301b63f5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/65ac58a16bdb4b53a523042fdac93d0f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a3d4e2256c41409dac77666420a1090e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9d17aea4989e45f8b19bc38da4d55400, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/7636995d7dbb41e49e430e8b22bfd7e4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/ef55c1ac011546629784af8483e8e016, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/90beaf7ad59d46af9cae8360caf11ada, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/45b6b97c024e42329b1c864f09f89ac1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8e1fc9af69794509b58eb7b4e5cf1f06, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9765c398a6104c0cab6366cdf5ed92dd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/cbdbd9550d774b2b9d6d73b1ef3eddd0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/eeda14965c73480e8740c055d6ea52ea] to archive 2024-11-14T19:35:52,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:35:52,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a7d0d76bf4a24d09b908b6d3e845ce74 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a7d0d76bf4a24d09b908b6d3e845ce74 2024-11-14T19:35:52,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/22be4069e35747f4ab52a111a1c3e884 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/22be4069e35747f4ab52a111a1c3e884 2024-11-14T19:35:52,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/e96b0406fc704087b141ba500a1dcbfa to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/e96b0406fc704087b141ba500a1dcbfa 2024-11-14T19:35:52,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/37c2b2a1fc1345bd8d15104f431801e3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/37c2b2a1fc1345bd8d15104f431801e3 2024-11-14T19:35:52,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/4afe457a5d144acc8e228bd230ed25d5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/4afe457a5d144acc8e228bd230ed25d5 2024-11-14T19:35:52,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b49f49b5395d418cb16bc8e29a982624 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b49f49b5395d418cb16bc8e29a982624 2024-11-14T19:35:52,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7311b53f95b404ab4d1805d1193f843 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7311b53f95b404ab4d1805d1193f843 2024-11-14T19:35:52,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/199cf73a0cd84bc7b913ed737a10bd4b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/199cf73a0cd84bc7b913ed737a10bd4b 2024-11-14T19:35:52,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8759b9faea804975b3ed3dccd60491f5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8759b9faea804975b3ed3dccd60491f5 2024-11-14T19:35:52,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/f630c30195d3481986bb9a9f7e10e9ad to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/f630c30195d3481986bb9a9f7e10e9ad 2024-11-14T19:35:52,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/1e5aaffdbfe64858a649cc8dd91674a8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/1e5aaffdbfe64858a649cc8dd91674a8 2024-11-14T19:35:52,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9db44299e1c2448cae23299789586960 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9db44299e1c2448cae23299789586960 2024-11-14T19:35:52,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/5af4ace65d574eecba69faee30761bee to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/5af4ace65d574eecba69faee30761bee 2024-11-14T19:35:52,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/89e296bbfb73423d95760f9eb3a043f2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/89e296bbfb73423d95760f9eb3a043f2 2024-11-14T19:35:52,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a2b04896158144b2840366d2d32d7ab5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a2b04896158144b2840366d2d32d7ab5 2024-11-14T19:35:52,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b2a93ac12c874d72b5718e17301b63f5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b2a93ac12c874d72b5718e17301b63f5 2024-11-14T19:35:52,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/65ac58a16bdb4b53a523042fdac93d0f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/65ac58a16bdb4b53a523042fdac93d0f 2024-11-14T19:35:52,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a3d4e2256c41409dac77666420a1090e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/a3d4e2256c41409dac77666420a1090e 2024-11-14T19:35:52,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9d17aea4989e45f8b19bc38da4d55400 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9d17aea4989e45f8b19bc38da4d55400 2024-11-14T19:35:52,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/7636995d7dbb41e49e430e8b22bfd7e4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/7636995d7dbb41e49e430e8b22bfd7e4 2024-11-14T19:35:52,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/ef55c1ac011546629784af8483e8e016 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/ef55c1ac011546629784af8483e8e016 2024-11-14T19:35:52,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/90beaf7ad59d46af9cae8360caf11ada to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/90beaf7ad59d46af9cae8360caf11ada 2024-11-14T19:35:52,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/45b6b97c024e42329b1c864f09f89ac1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/45b6b97c024e42329b1c864f09f89ac1 2024-11-14T19:35:52,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8e1fc9af69794509b58eb7b4e5cf1f06 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/8e1fc9af69794509b58eb7b4e5cf1f06 2024-11-14T19:35:52,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9765c398a6104c0cab6366cdf5ed92dd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/9765c398a6104c0cab6366cdf5ed92dd 2024-11-14T19:35:52,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/cbdbd9550d774b2b9d6d73b1ef3eddd0 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/cbdbd9550d774b2b9d6d73b1ef3eddd0 2024-11-14T19:35:52,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/eeda14965c73480e8740c055d6ea52ea to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/eeda14965c73480e8740c055d6ea52ea 2024-11-14T19:35:52,548 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/recovered.edits/396.seqid, newMaxSeqId=396, maxSeqId=4 2024-11-14T19:35:52,549 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. 2024-11-14T19:35:52,549 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 965abd24f852442dbb51955455ef7db5: Waiting for close lock at 1731612950611Running coprocessor pre-close hooks at 1731612950612 (+1 ms)Disabling compacts and flushes for region at 1731612950612Disabling writes for close at 1731612950612Obtaining lock to block concurrent updates at 1731612950612Preparing flush snapshotting stores in 965abd24f852442dbb51955455ef7db5 at 1731612950612Finished memstore snapshotting TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731612950612Flushing stores of TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5. at 1731612950613 (+1 ms)Flushing 965abd24f852442dbb51955455ef7db5/A: creating writer at 1731612950613Flushing mob file 965abd24f852442dbb51955455ef7db5/A: appending metadata at 1731612950641 (+28 ms)Flushing mob file 965abd24f852442dbb51955455ef7db5/A: closing flushed file at 1731612950641Flushing 965abd24f852442dbb51955455ef7db5/A: appending metadata at 1731612951051 (+410 ms)Flushing 965abd24f852442dbb51955455ef7db5/A: closing flushed file at 1731612951051Flushing 965abd24f852442dbb51955455ef7db5/B: creating writer at 1731612951461 (+410 ms)Flushing 965abd24f852442dbb51955455ef7db5/B: appending metadata at 1731612951464 (+3 ms)Flushing 965abd24f852442dbb51955455ef7db5/B: closing flushed file at 1731612951464Flushing 965abd24f852442dbb51955455ef7db5/C: creating writer at 1731612951873 (+409 ms)Flushing 965abd24f852442dbb51955455ef7db5/C: appending metadata at 1731612951876 (+3 ms)Flushing 965abd24f852442dbb51955455ef7db5/C: closing flushed file at 1731612951876Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e0fda91: reopening flushed file at 1731612952299 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52331634: reopening flushed file at 1731612952304 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd61a0f: reopening flushed file at 1731612952310 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 965abd24f852442dbb51955455ef7db5 in 1704ms, sequenceid=393, compaction requested=false at 1731612952316 (+6 ms)Writing region close event to WAL at 1731612952544 (+228 ms)Running coprocessor post-close hooks at 1731612952548 (+4 ms)Closed at 1731612952549 (+1 ms) 2024-11-14T19:35:52,552 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,552 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=965abd24f852442dbb51955455ef7db5, regionState=CLOSED 2024-11-14T19:35:52,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:35:52,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-14T19:35:52,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 965abd24f852442dbb51955455ef7db5, server=867b237d0fa7,38601,1731612889637 in 2.0960 sec 2024-11-14T19:35:52,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-14T19:35:52,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=965abd24f852442dbb51955455ef7db5, UNASSIGN in 2.1020 sec 2024-11-14T19:35:52,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-14T19:35:52,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1070 sec 2024-11-14T19:35:52,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612952562"}]},"ts":"1731612952562"} 2024-11-14T19:35:52,564 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T19:35:52,564 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T19:35:52,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1180 sec 2024-11-14T19:35:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-14T19:35:52,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:52,594 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T19:35:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:52,596 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T19:35:52,596 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:52,599 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,601 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/recovered.edits] 2024-11-14T19:35:52,606 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/683374359f1f4ce8940bf172a6757af7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/683374359f1f4ce8940bf172a6757af7 2024-11-14T19:35:52,607 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/a8afb326526b4911abaefa1ce9fe23d1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/A/a8afb326526b4911abaefa1ce9fe23d1 2024-11-14T19:35:52,610 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b72925a33c7b411680a63f502af01971 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/b72925a33c7b411680a63f502af01971 2024-11-14T19:35:52,612 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/d0772e99e9cb4be9a5f840fcac3393b8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/B/d0772e99e9cb4be9a5f840fcac3393b8 2024-11-14T19:35:52,615 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7ed0de5772a4928b8759c6f3720ff1a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/b7ed0de5772a4928b8759c6f3720ff1a 2024-11-14T19:35:52,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/c45d7f8d37f5473f86c4162381c7a25e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/C/c45d7f8d37f5473f86c4162381c7a25e 2024-11-14T19:35:52,622 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/recovered.edits/396.seqid to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5/recovered.edits/396.seqid 2024-11-14T19:35:52,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,623 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T19:35:52,624 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T19:35:52,625 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T19:35:52,633 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114025dbcba758343a69ee3ec42e6bdbf6d_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114025dbcba758343a69ee3ec42e6bdbf6d_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,636 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409263061f7ad443d87dd70a6e7c17e73_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111409263061f7ad443d87dd70a6e7c17e73_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,641 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410f0fd6d02c54f97b598992608edfc70_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111410f0fd6d02c54f97b598992608edfc70_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,643 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114313202b547504142a4212a09b9e125c4_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114313202b547504142a4212a09b9e125c4_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,646 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143b954fdd8380405ba148aa6bd2580960_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143b954fdd8380405ba148aa6bd2580960_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,648 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143bdb1755a14d45838899064494c3faa2_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143bdb1755a14d45838899064494c3faa2_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,650 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f23d2119cbf499b98080abe0acbb6e8_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143f23d2119cbf499b98080abe0acbb6e8_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,652 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144a2f5ccd130743038c436c027b7e55a0_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144a2f5ccd130743038c436c027b7e55a0_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,653 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b9c630e347c40cbbd6680f4f9c97824_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144b9c630e347c40cbbd6680f4f9c97824_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,656 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146409bfa8ff9648cfb7e4b8997df4c942_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146409bfa8ff9648cfb7e4b8997df4c942_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,658 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147264622641f743468c1485adc1381992_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147264622641f743468c1485adc1381992_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,660 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148109706809754114914be26bf5a48809_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148109706809754114914be26bf5a48809_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,662 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148533a0fd69554c71879064d286b8ad1e_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148533a0fd69554c71879064d286b8ad1e_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,663 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487384021877e477599f3be2f55cf7d32_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111487384021877e477599f3be2f55cf7d32_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,665 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148e643249e9894c5f82b4d41630270d87_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148e643249e9894c5f82b4d41630270d87_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,667 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149d8f8615506549818503dac0c941b111_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149d8f8615506549818503dac0c941b111_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,668 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114afbb892fe3e243f2ad76b5ac4038e5b7_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114afbb892fe3e243f2ad76b5ac4038e5b7_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,670 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bfcf86e33f5342579c93607bf03d4cb3_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114bfcf86e33f5342579c93607bf03d4cb3_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,671 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d2feb3b15e014f1fbce0d4cd6607ff58_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d2feb3b15e014f1fbce0d4cd6607ff58_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,673 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d53f20c527bf425e9beb4982075d2cf4_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114d53f20c527bf425e9beb4982075d2cf4_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,675 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f0f78936a98045febc79c64d7da6643c_965abd24f852442dbb51955455ef7db5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f0f78936a98045febc79c64d7da6643c_965abd24f852442dbb51955455ef7db5 2024-11-14T19:35:52,676 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T19:35:52,681 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:52,685 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T19:35:52,688 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T19:35:52,693 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:52,693 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T19:35:52,693 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731612952693"}]},"ts":"9223372036854775807"} 2024-11-14T19:35:52,697 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T19:35:52,697 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 965abd24f852442dbb51955455ef7db5, NAME => 'TestAcidGuarantees,,1731612918309.965abd24f852442dbb51955455ef7db5.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T19:35:52,697 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T19:35:52,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731612952697"}]},"ts":"9223372036854775807"} 2024-11-14T19:35:52,703 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T19:35:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T19:35:52,704 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:52,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 110 msec 2024-11-14T19:35:52,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-14T19:35:52,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T19:35:52,914 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:52,924 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 248), OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=570 (was 443) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5663 (was 5906) 2024-11-14T19:35:52,934 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=570, ProcessCount=11, AvailableMemoryMB=5663 2024-11-14T19:35:52,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:35:52,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:35:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T19:35:52,938 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T19:35:52,938 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:52,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-14T19:35:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-14T19:35:52,939 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T19:35:52,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742058_1234 (size=1061) 2024-11-14T19:35:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-14T19:35:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-14T19:35:53,350 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:35:53,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742059_1235 (size=53) 2024-11-14T19:35:53,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-14T19:35:53,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:35:53,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 47778d84645b4054cd2f50e381343edd, disabling compactions & flushes 2024-11-14T19:35:53,756 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:53,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:53,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. after waiting 0 ms 2024-11-14T19:35:53,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:53,756 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:53,756 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 47778d84645b4054cd2f50e381343edd: Waiting for close lock at 1731612953756Disabling compacts and flushes for region at 1731612953756Disabling writes for close at 1731612953756Writing region close event to WAL at 1731612953756Closed at 1731612953756 2024-11-14T19:35:53,758 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T19:35:53,758 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731612953758"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731612953758"}]},"ts":"1731612953758"} 2024-11-14T19:35:53,760 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T19:35:53,761 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T19:35:53,761 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612953761"}]},"ts":"1731612953761"} 2024-11-14T19:35:53,762 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T19:35:53,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, ASSIGN}] 2024-11-14T19:35:53,764 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, ASSIGN 2024-11-14T19:35:53,765 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, ASSIGN; state=OFFLINE, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=false 2024-11-14T19:35:53,915 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=47778d84645b4054cd2f50e381343edd, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:53,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, ASSIGN because future has completed 2024-11-14T19:35:53,917 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:35:54,072 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:54,073 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:35:54,073 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,073 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:35:54,073 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,073 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-14T19:35:54,075 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,076 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:54,076 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47778d84645b4054cd2f50e381343edd columnFamilyName A 2024-11-14T19:35:54,076 DEBUG [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:54,077 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.HStore(327): Store=47778d84645b4054cd2f50e381343edd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:54,077 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,078 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:54,078 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47778d84645b4054cd2f50e381343edd columnFamilyName B 2024-11-14T19:35:54,078 DEBUG [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:54,078 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.HStore(327): Store=47778d84645b4054cd2f50e381343edd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:54,078 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,079 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:35:54,080 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47778d84645b4054cd2f50e381343edd columnFamilyName C 2024-11-14T19:35:54,080 DEBUG [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:35:54,080 INFO [StoreOpener-47778d84645b4054cd2f50e381343edd-1 {}] regionserver.HStore(327): Store=47778d84645b4054cd2f50e381343edd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:35:54,080 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:54,080 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,081 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,081 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,083 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,083 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,083 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:35:54,084 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,086 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:35:54,087 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 47778d84645b4054cd2f50e381343edd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64149683, jitterRate=-0.0440952330827713}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:35:54,087 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:54,087 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 47778d84645b4054cd2f50e381343edd: Running coprocessor pre-open hook at 1731612954073Writing region info on filesystem at 1731612954074 (+1 ms)Initializing all the Stores at 1731612954074Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612954074Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612954075 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612954075Cleaning up temporary data from old regions at 1731612954083 (+8 ms)Running coprocessor post-open hooks at 1731612954087 (+4 ms)Region opened successfully at 1731612954087 2024-11-14T19:35:54,089 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., pid=65, masterSystemTime=1731612954069 2024-11-14T19:35:54,091 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:54,092 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:54,092 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=47778d84645b4054cd2f50e381343edd, regionState=OPEN, openSeqNum=2, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:54,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:35:54,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-14T19:35:54,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 in 177 msec 2024-11-14T19:35:54,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-14T19:35:54,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, ASSIGN in 334 msec 2024-11-14T19:35:54,100 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T19:35:54,101 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612954101"}]},"ts":"1731612954101"} 2024-11-14T19:35:54,102 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T19:35:54,104 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T19:35:54,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-14T19:35:55,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-14T19:35:55,084 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:55,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74cbe010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,087 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ed6fbfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,089 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154ab8f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,090 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,092 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,093 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T19:35:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 867b237d0fa7,38019,1731612888274 2024-11-14T19:35:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7976b547 2024-11-14T19:35:55,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T19:35:55,095 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T19:35:55,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1c5003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345bebf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,099 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a357a23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cde69b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,102 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79726149, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,102 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,103 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cdffc90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cf162e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,106 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73031283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,107 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c9212f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,108 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240c3b1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,109 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,109 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,110 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e8b30fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,111 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13cb452a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5faf75fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,116 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523408bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,118 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7336a490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d01f46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,124 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b48620e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,127 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64fb4c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bdd3a3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,132 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a1ae95b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,134 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134c62cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fafaf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,140 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7e2220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,142 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a078dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16f507a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,145 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9ebe3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,147 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@745383e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35038a97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:35:55,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:35:55,155 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:35:55,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:35:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:35:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ef5e5f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:35:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:35:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:35:55,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:35:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edfcaf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:35:55,163 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,163 DEBUG [Thread-1102 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-14T19:35:55,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T19:35:55,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:55,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:55,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:55,187 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,190 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:55,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:35:55,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:35:55,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:55,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:35:55,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:55,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:35:55,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:55,208 DEBUG [Thread-1098 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,219 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,233 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,245 DEBUG [Thread-1100 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/1fef07a801ed4eae8bf28d3b86e867e4 is 50, key is test_row_0/A:col10/1731612955190/Put/seqid=0 2024-11-14T19:35:55,259 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,261 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,266 DEBUG [Thread-1104 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:35:55,267 DEBUG [Thread-1104 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:35:55,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731612955228, max timestamp=1731612955265Num uniques 859; 2024-11-14T19:35:55,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:35:55,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731612955228, max timestamp=1731612955265Num uniques 859; 2024-11-14T19:35:55,271 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:35:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731612955228, max timestamp=1731612955265Num uniques 859; 2024-11-14T19:35:55,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:35:55,275 DEBUG [Thread-1104 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:35:55,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:35:55,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:35:55,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T19:35:55,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742060_1236 (size=14341) 2024-11-14T19:35:55,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613015304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731612955298, max timestamp=1731612955322Num uniques 1068; 2024-11-14T19:35:55,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731612955298, max timestamp=1731612955322Num uniques 1068; 2024-11-14T19:35:55,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T19:35:55,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-14T19:35:55,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612955298, max timestamp=1731612955322Num uniques 1068; 2024-11-14T19:35:55,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-14T19:35:55,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613015376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613015376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613015378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613015381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613015433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:55,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:55,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T19:35:55,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613015494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613015494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613015494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613015494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:55,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:55,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613015643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/1fef07a801ed4eae8bf28d3b86e867e4 2024-11-14T19:35:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613015703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613015704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613015704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613015704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:55,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/049e18e3951343789a51431b545fef1e is 50, key is test_row_0/B:col10/1731612955190/Put/seqid=0 2024-11-14T19:35:55,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742061_1237 (size=12001) 2024-11-14T19:35:55,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T19:35:55,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:55,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:55,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613015963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:55,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613016023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613016023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613016023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613016023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:56,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:56,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/049e18e3951343789a51431b545fef1e 2024-11-14T19:35:56,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/437e8970fd164a1eaad5255a09f52843 is 50, key is test_row_0/C:col10/1731612955190/Put/seqid=0 2024-11-14T19:35:56,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:56,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:56,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:56,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:56,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742062_1238 (size=12001) 2024-11-14T19:35:56,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/437e8970fd164a1eaad5255a09f52843 2024-11-14T19:35:56,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/1fef07a801ed4eae8bf28d3b86e867e4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1fef07a801ed4eae8bf28d3b86e867e4 2024-11-14T19:35:56,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1fef07a801ed4eae8bf28d3b86e867e4, entries=200, sequenceid=11, filesize=14.0 K 2024-11-14T19:35:56,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/049e18e3951343789a51431b545fef1e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/049e18e3951343789a51431b545fef1e 2024-11-14T19:35:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T19:35:56,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/049e18e3951343789a51431b545fef1e, entries=150, sequenceid=11, filesize=11.7 K 2024-11-14T19:35:56,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/437e8970fd164a1eaad5255a09f52843 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/437e8970fd164a1eaad5255a09f52843 2024-11-14T19:35:56,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/437e8970fd164a1eaad5255a09f52843, entries=150, sequenceid=11, filesize=11.7 K 2024-11-14T19:35:56,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 47778d84645b4054cd2f50e381343edd in 1138ms, sequenceid=11, compaction requested=false 2024-11-14T19:35:56,338 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T19:35:56,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:35:56,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-14T19:35:56,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:56,414 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-14T19:35:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:35:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:35:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:35:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:35:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/748c424ad65640d09baccce989ae250b is 50, key is test_row_0/A:col10/1731612955371/Put/seqid=0 2024-11-14T19:35:56,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742063_1239 (size=12001) 2024-11-14T19:35:56,463 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/748c424ad65640d09baccce989ae250b 2024-11-14T19:35:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:56,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:56,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/22f43cbc77c2426292b8f8790a3cb9be is 50, key is test_row_0/B:col10/1731612955371/Put/seqid=0 2024-11-14T19:35:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742064_1240 (size=12001) 2024-11-14T19:35:56,516 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/22f43cbc77c2426292b8f8790a3cb9be 2024-11-14T19:35:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/b8e16ceddbf74ea5a9715442f2ae7dc8 is 50, key is test_row_0/C:col10/1731612955371/Put/seqid=0 2024-11-14T19:35:56,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613016534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613016535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613016542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613016542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613016543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742065_1241 (size=12001) 2024-11-14T19:35:56,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613016653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613016653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613016654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613016863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613016863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:56,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613016863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:56,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,009 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/b8e16ceddbf74ea5a9715442f2ae7dc8 2024-11-14T19:35:57,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/748c424ad65640d09baccce989ae250b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/748c424ad65640d09baccce989ae250b 2024-11-14T19:35:57,020 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/748c424ad65640d09baccce989ae250b, entries=150, sequenceid=64, filesize=11.7 K 2024-11-14T19:35:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/22f43cbc77c2426292b8f8790a3cb9be as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/22f43cbc77c2426292b8f8790a3cb9be 2024-11-14T19:35:57,027 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/22f43cbc77c2426292b8f8790a3cb9be, entries=150, sequenceid=64, filesize=11.7 K 2024-11-14T19:35:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/b8e16ceddbf74ea5a9715442f2ae7dc8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8e16ceddbf74ea5a9715442f2ae7dc8 2024-11-14T19:35:57,035 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8e16ceddbf74ea5a9715442f2ae7dc8, entries=150, sequenceid=64, filesize=11.7 K 2024-11-14T19:35:57,038 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=53.67 KB/54960 for 47778d84645b4054cd2f50e381343edd in 623ms, sequenceid=64, compaction requested=false 2024-11-14T19:35:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:35:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-14T19:35:57,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-14T19:35:57,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-14T19:35:57,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8600 sec 2024-11-14T19:35:57,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8680 sec 2024-11-14T19:35:57,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:35:57,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:35:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:35:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:35:57,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:57,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/9d160d7ee9494203aa82cb915d1894f0 is 50, key is test_row_0/A:col10/1731612956497/Put/seqid=0 2024-11-14T19:35:57,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742066_1242 (size=12001) 2024-11-14T19:35:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613017263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613017264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613017266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-14T19:35:57,334 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:57,336 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-14T19:35:57,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:57,339 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:57,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T19:35:57,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613017376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613017376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613017384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T19:35:57,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T19:35:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613017553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,556 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 47778d84645b4054cd2f50e381343edd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,557 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613017563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,566 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 47778d84645b4054cd2f50e381343edd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:57,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613017583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613017583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613017593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/9d160d7ee9494203aa82cb915d1894f0 2024-11-14T19:35:57,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T19:35:57,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T19:35:57,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/f5529468bb5b4e63b01171115f5657fd is 50, key is test_row_0/B:col10/1731612956497/Put/seqid=0 2024-11-14T19:35:57,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742067_1243 (size=12001) 2024-11-14T19:35:57,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/f5529468bb5b4e63b01171115f5657fd 2024-11-14T19:35:57,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8c01d16d84c349849862a20d846b0ca5 is 50, key is test_row_0/C:col10/1731612956497/Put/seqid=0 2024-11-14T19:35:57,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742068_1244 (size=12001) 2024-11-14T19:35:57,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T19:35:57,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:57,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613017893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613017893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613017904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:57,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T19:35:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:57,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T19:35:58,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T19:35:58,108 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T19:35:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:58,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:58,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8c01d16d84c349849862a20d846b0ca5 2024-11-14T19:35:58,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/9d160d7ee9494203aa82cb915d1894f0 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9d160d7ee9494203aa82cb915d1894f0 2024-11-14T19:35:58,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9d160d7ee9494203aa82cb915d1894f0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T19:35:58,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/f5529468bb5b4e63b01171115f5657fd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/f5529468bb5b4e63b01171115f5657fd 2024-11-14T19:35:58,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/f5529468bb5b4e63b01171115f5657fd, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T19:35:58,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8c01d16d84c349849862a20d846b0ca5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8c01d16d84c349849862a20d846b0ca5 2024-11-14T19:35:58,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8c01d16d84c349849862a20d846b0ca5, entries=150, sequenceid=76, filesize=11.7 K 2024-11-14T19:35:58,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 47778d84645b4054cd2f50e381343edd in 1017ms, sequenceid=76, compaction requested=true 2024-11-14T19:35:58,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:35:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:35:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:58,194 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:35:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:58,194 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:35:58,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:58,195 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:58,195 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/B is initiating minor compaction (all files) 2024-11-14T19:35:58,195 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/B in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:58,195 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/049e18e3951343789a51431b545fef1e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/22f43cbc77c2426292b8f8790a3cb9be, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/f5529468bb5b4e63b01171115f5657fd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=35.2 K 2024-11-14T19:35:58,196 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:58,196 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/A is initiating minor compaction (all files) 2024-11-14T19:35:58,196 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 049e18e3951343789a51431b545fef1e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731612955182 2024-11-14T19:35:58,196 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/A in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:58,196 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1fef07a801ed4eae8bf28d3b86e867e4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/748c424ad65640d09baccce989ae250b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9d160d7ee9494203aa82cb915d1894f0] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=37.4 K 2024-11-14T19:35:58,196 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 22f43cbc77c2426292b8f8790a3cb9be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731612955371 2024-11-14T19:35:58,197 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f5529468bb5b4e63b01171115f5657fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731612956497 2024-11-14T19:35:58,197 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fef07a801ed4eae8bf28d3b86e867e4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731612955182 2024-11-14T19:35:58,197 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 748c424ad65640d09baccce989ae250b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731612955371 2024-11-14T19:35:58,198 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d160d7ee9494203aa82cb915d1894f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731612956497 2024-11-14T19:35:58,215 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#B#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:58,216 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/2fa20c780ae24a42bee80f5e46ecff6b is 50, key is test_row_0/B:col10/1731612956497/Put/seqid=0 2024-11-14T19:35:58,228 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#A#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:58,229 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/cf2e31b6973f41a0abfa6fc09d838b13 is 50, key is test_row_0/A:col10/1731612956497/Put/seqid=0 2024-11-14T19:35:58,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-14T19:35:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:58,263 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:35:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:35:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:35:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:35:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:58,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742070_1246 (size=12104) 2024-11-14T19:35:58,287 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/cf2e31b6973f41a0abfa6fc09d838b13 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf2e31b6973f41a0abfa6fc09d838b13 2024-11-14T19:35:58,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742069_1245 (size=12104) 2024-11-14T19:35:58,297 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/A of 47778d84645b4054cd2f50e381343edd into cf2e31b6973f41a0abfa6fc09d838b13(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:58,297 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:35:58,297 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/A, priority=13, startTime=1731612958193; duration=0sec 2024-11-14T19:35:58,297 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:35:58,297 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:A 2024-11-14T19:35:58,297 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:35:58,299 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:35:58,299 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/C is initiating minor compaction (all files) 2024-11-14T19:35:58,299 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/C in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:58,300 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/437e8970fd164a1eaad5255a09f52843, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8e16ceddbf74ea5a9715442f2ae7dc8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8c01d16d84c349849862a20d846b0ca5] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=35.2 K 2024-11-14T19:35:58,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/5e261faa82b34dd5b67d6afd48def56c is 50, key is test_row_0/A:col10/1731612957264/Put/seqid=0 2024-11-14T19:35:58,301 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 437e8970fd164a1eaad5255a09f52843, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731612955182 2024-11-14T19:35:58,302 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8e16ceddbf74ea5a9715442f2ae7dc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731612955371 2024-11-14T19:35:58,303 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c01d16d84c349849862a20d846b0ca5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731612956497 2024-11-14T19:35:58,326 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#C#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:35:58,326 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/f6bc3f92c0aa4a07ba89ae13d0123d4b is 50, key is test_row_0/C:col10/1731612956497/Put/seqid=0 2024-11-14T19:35:58,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742071_1247 (size=12001) 2024-11-14T19:35:58,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742072_1248 (size=12104) 2024-11-14T19:35:58,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613018429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613018436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613018436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T19:35:58,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613018544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613018554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613018553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,561 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,702 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/2fa20c780ae24a42bee80f5e46ecff6b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/2fa20c780ae24a42bee80f5e46ecff6b 2024-11-14T19:35:58,708 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/B of 47778d84645b4054cd2f50e381343edd into 2fa20c780ae24a42bee80f5e46ecff6b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:58,709 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:35:58,709 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/B, priority=13, startTime=1731612958194; duration=0sec 2024-11-14T19:35:58,709 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:58,709 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:B 2024-11-14T19:35:58,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613018753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,757 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/5e261faa82b34dd5b67d6afd48def56c 2024-11-14T19:35:58,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613018763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613018763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:58,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/62260b3b6b704705ad92b307a9f42e1f is 50, key is test_row_0/B:col10/1731612957264/Put/seqid=0 2024-11-14T19:35:58,808 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/f6bc3f92c0aa4a07ba89ae13d0123d4b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f6bc3f92c0aa4a07ba89ae13d0123d4b 2024-11-14T19:35:58,814 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/C of 47778d84645b4054cd2f50e381343edd into f6bc3f92c0aa4a07ba89ae13d0123d4b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:35:58,814 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:35:58,814 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/C, priority=13, startTime=1731612958194; duration=0sec 2024-11-14T19:35:58,814 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:35:58,814 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:C 2024-11-14T19:35:58,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742073_1249 (size=12001) 2024-11-14T19:35:58,818 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/62260b3b6b704705ad92b307a9f42e1f 2024-11-14T19:35:58,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/4d039298d96248fba61b900df355b93b is 50, key is test_row_0/C:col10/1731612957264/Put/seqid=0 2024-11-14T19:35:58,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742074_1250 (size=12001) 2024-11-14T19:35:58,851 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/4d039298d96248fba61b900df355b93b 2024-11-14T19:35:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/5e261faa82b34dd5b67d6afd48def56c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/5e261faa82b34dd5b67d6afd48def56c 2024-11-14T19:35:58,867 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/5e261faa82b34dd5b67d6afd48def56c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T19:35:58,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/62260b3b6b704705ad92b307a9f42e1f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/62260b3b6b704705ad92b307a9f42e1f 2024-11-14T19:35:58,877 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/62260b3b6b704705ad92b307a9f42e1f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T19:35:58,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/4d039298d96248fba61b900df355b93b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/4d039298d96248fba61b900df355b93b 2024-11-14T19:35:58,886 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/4d039298d96248fba61b900df355b93b, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T19:35:58,888 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 47778d84645b4054cd2f50e381343edd in 624ms, sequenceid=96, compaction requested=false 2024-11-14T19:35:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:35:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:58,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-14T19:35:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-14T19:35:58,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-14T19:35:58,893 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-11-14T19:35:58,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.5590 sec 2024-11-14T19:35:59,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T19:35:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:35:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:35:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:35:59,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:35:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:35:59,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/9f6aa9994776404b8811c0dfe07e903c is 50, key is test_row_0/A:col10/1731612958433/Put/seqid=0 2024-11-14T19:35:59,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742075_1251 (size=12001) 2024-11-14T19:35:59,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613019125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613019129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613019134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613019243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613019243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613019253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613019453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613019453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613019463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-14T19:35:59,493 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:35:59,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:35:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-14T19:35:59,497 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:35:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T19:35:59,498 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:35:59,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:35:59,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/9f6aa9994776404b8811c0dfe07e903c 2024-11-14T19:35:59,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/61af9258b5d044d38d61749ec43951aa is 50, key is test_row_0/B:col10/1731612958433/Put/seqid=0 2024-11-14T19:35:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613019583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613019583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,588 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 47778d84645b4054cd2f50e381343edd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,589 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 47778d84645b4054cd2f50e381343edd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:35:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T19:35:59,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742076_1252 (size=12001) 2024-11-14T19:35:59,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T19:35:59,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:59,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:59,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:59,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613019763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613019763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:35:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613019773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:35:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:35:59,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:35:59,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T19:35:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:59,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T19:35:59,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T19:35:59,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:59,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:35:59,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:35:59,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:35:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/61af9258b5d044d38d61749ec43951aa 2024-11-14T19:36:00,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1cc8eb600aff47dc9bfd933ca26508f8 is 50, key is test_row_0/C:col10/1731612958433/Put/seqid=0 2024-11-14T19:36:00,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742077_1253 (size=12001) 2024-11-14T19:36:00,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T19:36:00,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:00,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T19:36:00,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T19:36:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:00,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613020273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:00,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613020273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:00,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613020293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:00,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T19:36:00,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:00,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:00,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1cc8eb600aff47dc9bfd933ca26508f8 2024-11-14T19:36:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/9f6aa9994776404b8811c0dfe07e903c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9f6aa9994776404b8811c0dfe07e903c 2024-11-14T19:36:00,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9f6aa9994776404b8811c0dfe07e903c, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T19:36:00,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/61af9258b5d044d38d61749ec43951aa as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/61af9258b5d044d38d61749ec43951aa 2024-11-14T19:36:00,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/61af9258b5d044d38d61749ec43951aa, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T19:36:00,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1cc8eb600aff47dc9bfd933ca26508f8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1cc8eb600aff47dc9bfd933ca26508f8 2024-11-14T19:36:00,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1cc8eb600aff47dc9bfd933ca26508f8, entries=150, sequenceid=113, filesize=11.7 K 2024-11-14T19:36:00,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 47778d84645b4054cd2f50e381343edd in 1478ms, sequenceid=113, compaction requested=true 2024-11-14T19:36:00,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:00,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:00,557 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:00,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:00,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:00,557 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:00,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:00,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:00,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:00,558 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:00,559 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/A is initiating minor compaction (all files) 2024-11-14T19:36:00,559 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/A in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,559 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:00,559 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/B is initiating minor compaction (all files) 2024-11-14T19:36:00,559 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf2e31b6973f41a0abfa6fc09d838b13, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/5e261faa82b34dd5b67d6afd48def56c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9f6aa9994776404b8811c0dfe07e903c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=35.3 K 2024-11-14T19:36:00,559 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/B in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,559 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/2fa20c780ae24a42bee80f5e46ecff6b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/62260b3b6b704705ad92b307a9f42e1f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/61af9258b5d044d38d61749ec43951aa] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=35.3 K 2024-11-14T19:36:00,559 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2e31b6973f41a0abfa6fc09d838b13, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731612956497 2024-11-14T19:36:00,560 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa20c780ae24a42bee80f5e46ecff6b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731612956497 2024-11-14T19:36:00,560 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e261faa82b34dd5b67d6afd48def56c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731612957223 2024-11-14T19:36:00,560 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f6aa9994776404b8811c0dfe07e903c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731612958433 2024-11-14T19:36:00,560 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 62260b3b6b704705ad92b307a9f42e1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731612957223 2024-11-14T19:36:00,561 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 61af9258b5d044d38d61749ec43951aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731612958433 2024-11-14T19:36:00,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-14T19:36:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,575 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:00,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:00,584 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#A#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:00,585 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/c9b40eda811543e69bdb894c654261e8 is 50, key is test_row_0/A:col10/1731612958433/Put/seqid=0 2024-11-14T19:36:00,593 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:00,593 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/246d4ba0500e432fa41032fa28f19f91 is 50, key is test_row_0/B:col10/1731612958433/Put/seqid=0 2024-11-14T19:36:00,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/07ef55d6f1f34e33a48b80268882fad8 is 50, key is test_row_0/A:col10/1731612959132/Put/seqid=0 2024-11-14T19:36:00,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742079_1255 (size=12207) 2024-11-14T19:36:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T19:36:00,639 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/246d4ba0500e432fa41032fa28f19f91 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/246d4ba0500e432fa41032fa28f19f91 2024-11-14T19:36:00,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742078_1254 (size=12207) 2024-11-14T19:36:00,652 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/B of 47778d84645b4054cd2f50e381343edd into 246d4ba0500e432fa41032fa28f19f91(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:00,652 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:00,652 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/B, priority=13, startTime=1731612960557; duration=0sec 2024-11-14T19:36:00,652 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:00,652 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:B 2024-11-14T19:36:00,652 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:00,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742080_1256 (size=12101) 2024-11-14T19:36:00,656 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:00,656 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/C is initiating minor compaction (all files) 2024-11-14T19:36:00,656 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/C in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:00,656 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f6bc3f92c0aa4a07ba89ae13d0123d4b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/4d039298d96248fba61b900df355b93b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1cc8eb600aff47dc9bfd933ca26508f8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=35.3 K 2024-11-14T19:36:00,657 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f6bc3f92c0aa4a07ba89ae13d0123d4b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731612956497 2024-11-14T19:36:00,658 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d039298d96248fba61b900df355b93b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731612957223 2024-11-14T19:36:00,659 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cc8eb600aff47dc9bfd933ca26508f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731612958433 2024-11-14T19:36:00,672 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/c9b40eda811543e69bdb894c654261e8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/c9b40eda811543e69bdb894c654261e8 2024-11-14T19:36:00,693 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:00,693 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/6d3647fdd358416987c2ac3cde6dd8e1 is 50, key is test_row_0/C:col10/1731612958433/Put/seqid=0 2024-11-14T19:36:00,698 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/A of 47778d84645b4054cd2f50e381343edd into c9b40eda811543e69bdb894c654261e8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:00,698 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:00,698 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/A, priority=13, startTime=1731612960557; duration=0sec 2024-11-14T19:36:00,698 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:00,698 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:A 2024-11-14T19:36:00,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742081_1257 (size=12207) 2024-11-14T19:36:00,722 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/6d3647fdd358416987c2ac3cde6dd8e1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6d3647fdd358416987c2ac3cde6dd8e1 2024-11-14T19:36:00,733 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/C of 47778d84645b4054cd2f50e381343edd into 6d3647fdd358416987c2ac3cde6dd8e1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:00,733 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:00,733 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/C, priority=13, startTime=1731612960557; duration=0sec 2024-11-14T19:36:00,734 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:00,734 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:C 2024-11-14T19:36:01,056 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/07ef55d6f1f34e33a48b80268882fad8 2024-11-14T19:36:01,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/3aeac0165a2c47a1a5198e8a75d411d4 is 50, key is test_row_0/B:col10/1731612959132/Put/seqid=0 2024-11-14T19:36:01,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742082_1258 (size=12101) 2024-11-14T19:36:01,116 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/3aeac0165a2c47a1a5198e8a75d411d4 2024-11-14T19:36:01,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/ebf16398bb7e4b95a88dae0daf9b8504 is 50, key is test_row_0/C:col10/1731612959132/Put/seqid=0 2024-11-14T19:36:01,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742083_1259 (size=12101) 2024-11-14T19:36:01,182 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/ebf16398bb7e4b95a88dae0daf9b8504 2024-11-14T19:36:01,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/07ef55d6f1f34e33a48b80268882fad8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/07ef55d6f1f34e33a48b80268882fad8 2024-11-14T19:36:01,207 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/07ef55d6f1f34e33a48b80268882fad8, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T19:36:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/3aeac0165a2c47a1a5198e8a75d411d4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3aeac0165a2c47a1a5198e8a75d411d4 2024-11-14T19:36:01,220 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3aeac0165a2c47a1a5198e8a75d411d4, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T19:36:01,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/ebf16398bb7e4b95a88dae0daf9b8504 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/ebf16398bb7e4b95a88dae0daf9b8504 2024-11-14T19:36:01,233 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/ebf16398bb7e4b95a88dae0daf9b8504, entries=150, sequenceid=131, filesize=11.8 K 2024-11-14T19:36:01,235 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 47778d84645b4054cd2f50e381343edd in 659ms, sequenceid=131, compaction requested=false 2024-11-14T19:36:01,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:01,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:01,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-14T19:36:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-14T19:36:01,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-14T19:36:01,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7390 sec 2024-11-14T19:36:01,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7460 sec 2024-11-14T19:36:01,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:01,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:36:01,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:01,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:01,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:01,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:01,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:01,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:01,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/cf46fd9a17c04715979e950c383e69d2 is 50, key is test_row_0/A:col10/1731612961294/Put/seqid=0 2024-11-14T19:36:01,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742084_1260 (size=14541) 2024-11-14T19:36:01,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612961333, max timestamp=1731612961344Num uniques 859; 2024-11-14T19:36:01,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612961333, max timestamp=1731612961344Num uniques 859; 2024-11-14T19:36:01,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612961333, max timestamp=1731612961344Num uniques 859; 2024-11-14T19:36:01,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:36:01,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:36:01,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:36:01,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613021436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613021441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613021445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613021553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613021553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613021564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-14T19:36:01,643 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:01,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-14T19:36:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:01,645 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:01,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:01,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:01,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/cf46fd9a17c04715979e950c383e69d2 2024-11-14T19:36:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:01,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/15850e53dea94e3a8e4f5dce08b739ff is 50, key is test_row_0/B:col10/1731612961294/Put/seqid=0 2024-11-14T19:36:01,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742085_1261 (size=12151) 2024-11-14T19:36:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613021763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613021763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613021772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:01,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:01,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:01,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:01,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:01,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:01,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:01,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:02,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613022073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613022073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613022083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:02,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:02,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:02,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/15850e53dea94e3a8e4f5dce08b739ff 2024-11-14T19:36:02,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8d72ac7181b049cc8088f470abdb0546 is 50, key is test_row_0/C:col10/1731612961294/Put/seqid=0 2024-11-14T19:36:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742086_1262 (size=12151) 2024-11-14T19:36:02,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:02,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:02,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:02,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:02,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:02,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8d72ac7181b049cc8088f470abdb0546 2024-11-14T19:36:02,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/cf46fd9a17c04715979e950c383e69d2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf46fd9a17c04715979e950c383e69d2 2024-11-14T19:36:02,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf46fd9a17c04715979e950c383e69d2, entries=200, sequenceid=144, filesize=14.2 K 2024-11-14T19:36:02,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/15850e53dea94e3a8e4f5dce08b739ff as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/15850e53dea94e3a8e4f5dce08b739ff 2024-11-14T19:36:02,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/15850e53dea94e3a8e4f5dce08b739ff, entries=150, sequenceid=144, filesize=11.9 K 2024-11-14T19:36:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8d72ac7181b049cc8088f470abdb0546 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8d72ac7181b049cc8088f470abdb0546 2024-11-14T19:36:02,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613022583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:02,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:02,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8d72ac7181b049cc8088f470abdb0546, entries=150, sequenceid=144, filesize=11.9 K 2024-11-14T19:36:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613022583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:02,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 47778d84645b4054cd2f50e381343edd in 1294ms, sequenceid=144, compaction requested=true 2024-11-14T19:36:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:02,589 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:02,589 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:02,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:02,590 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38849 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:02,590 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:02,590 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/A is initiating minor compaction (all files) 2024-11-14T19:36:02,590 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/B is initiating minor compaction (all files) 2024-11-14T19:36:02,591 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/B in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,591 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/A in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,591 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/246d4ba0500e432fa41032fa28f19f91, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3aeac0165a2c47a1a5198e8a75d411d4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/15850e53dea94e3a8e4f5dce08b739ff] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=35.6 K 2024-11-14T19:36:02,591 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/c9b40eda811543e69bdb894c654261e8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/07ef55d6f1f34e33a48b80268882fad8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf46fd9a17c04715979e950c383e69d2] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=37.9 K 2024-11-14T19:36:02,591 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 246d4ba0500e432fa41032fa28f19f91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731612958433 2024-11-14T19:36:02,591 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9b40eda811543e69bdb894c654261e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731612958433 2024-11-14T19:36:02,591 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aeac0165a2c47a1a5198e8a75d411d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731612959109 2024-11-14T19:36:02,591 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ef55d6f1f34e33a48b80268882fad8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731612959109 2024-11-14T19:36:02,591 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 15850e53dea94e3a8e4f5dce08b739ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731612961288 2024-11-14T19:36:02,592 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf46fd9a17c04715979e950c383e69d2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731612961288 2024-11-14T19:36:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:02,599 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#B#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:02,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612961381, max timestamp=1731612962595Num uniques 1018; 2024-11-14T19:36:02,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612961381, max timestamp=1731612962595Num uniques 1018; 2024-11-14T19:36:02,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:36:02,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:36:02,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612961381, max timestamp=1731612962595Num uniques 1018; 2024-11-14T19:36:02,599 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:36:02,600 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/4055822feaf14deb80d77f321bbc2339 is 50, key is test_row_0/B:col10/1731612961294/Put/seqid=0 2024-11-14T19:36:02,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=26.84 KB heapSize=65.39 KB 2024-11-14T19:36:02,600 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:02,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:02,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:02,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:02,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:02,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:02,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/58864f773e3e40ef942449041a26df70 is 50, key is test_row_0/A:col10/1731612962595/Put/seqid=0 2024-11-14T19:36:02,617 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#A#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:02,617 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/451e98dd6bd047a4967d67bfcda87238 is 50, key is test_row_0/A:col10/1731612961294/Put/seqid=0 2024-11-14T19:36:02,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742087_1263 (size=12459) 2024-11-14T19:36:02,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742088_1264 (size=12151) 2024-11-14T19:36:02,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742089_1265 (size=12459) 2024-11-14T19:36:02,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612962658, max timestamp=1731612962676Num uniques 859; 2024-11-14T19:36:02,678 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:36:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612962658, max timestamp=1731612962676Num uniques 859; 2024-11-14T19:36:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:36:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731612962658, max timestamp=1731612962676Num uniques 859; 2024-11-14T19:36:02,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:36:02,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612962741, max timestamp=1731612962762Num uniques 1018; 2024-11-14T19:36:02,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612962741, max timestamp=1731612962762Num uniques 1018; 2024-11-14T19:36:02,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731612962741, max timestamp=1731612962762Num uniques 1018; 2024-11-14T19:36:02,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:36:02,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:36:02,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-14T19:36:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:02,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731612962837, max timestamp=1731612962854Num uniques 1027; 2024-11-14T19:36:02,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731612962837, max timestamp=1731612962854Num uniques 1027; 2024-11-14T19:36:02,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731612962837, max timestamp=1731612962854Num uniques 1027; 2024-11-14T19:36:02,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T19:36:02,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T19:36:02,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-14T19:36:02,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:02,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:02,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612962921, max timestamp=1731612962938Num uniques 1036; 2024-11-14T19:36:02,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T19:36:02,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612962921, max timestamp=1731612962938Num uniques 1036; 2024-11-14T19:36:02,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T19:36:02,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731612962921, max timestamp=1731612962938Num uniques 1036; 2024-11-14T19:36:02,946 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-14T19:36:03,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731612962988, max timestamp=1731612963018Num uniques 1045; 2024-11-14T19:36:03,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T19:36:03,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731612962988, max timestamp=1731612963018Num uniques 1045; 2024-11-14T19:36:03,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731612962988, max timestamp=1731612963018Num uniques 1045; 2024-11-14T19:36:03,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T19:36:03,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-14T19:36:03,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/58864f773e3e40ef942449041a26df70 2024-11-14T19:36:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,025 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/4055822feaf14deb80d77f321bbc2339 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/4055822feaf14deb80d77f321bbc2339 2024-11-14T19:36:03,033 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/B of 47778d84645b4054cd2f50e381343edd into 4055822feaf14deb80d77f321bbc2339(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:03,033 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:03,033 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/B, priority=13, startTime=1731612962589; duration=0sec 2024-11-14T19:36:03,033 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:03,034 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:B 2024-11-14T19:36:03,034 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:03,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/3f03156399e647e78dbcecdd8f2fc3d2 is 50, key is test_row_0/B:col10/1731612962595/Put/seqid=0 2024-11-14T19:36:03,034 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/451e98dd6bd047a4967d67bfcda87238 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/451e98dd6bd047a4967d67bfcda87238 2024-11-14T19:36:03,035 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:03,035 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/C is initiating minor compaction (all files) 2024-11-14T19:36:03,035 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/C in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,035 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6d3647fdd358416987c2ac3cde6dd8e1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/ebf16398bb7e4b95a88dae0daf9b8504, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8d72ac7181b049cc8088f470abdb0546] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=35.6 K 2024-11-14T19:36:03,036 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d3647fdd358416987c2ac3cde6dd8e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731612958433 2024-11-14T19:36:03,036 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf16398bb7e4b95a88dae0daf9b8504, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731612959109 2024-11-14T19:36:03,037 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d72ac7181b049cc8088f470abdb0546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731612961288 2024-11-14T19:36:03,040 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/A of 47778d84645b4054cd2f50e381343edd into 451e98dd6bd047a4967d67bfcda87238(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:03,040 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:03,040 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/A, priority=13, startTime=1731612962589; duration=0sec 2024-11-14T19:36:03,040 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:03,040 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:A 2024-11-14T19:36:03,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742090_1266 (size=12151) 2024-11-14T19:36:03,049 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#C#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:03,049 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/54601378d8474ee08e4523c04f882635 is 50, key is test_row_0/C:col10/1731612961294/Put/seqid=0 2024-11-14T19:36:03,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742091_1267 (size=12459) 2024-11-14T19:36:03,060 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/54601378d8474ee08e4523c04f882635 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/54601378d8474ee08e4523c04f882635 2024-11-14T19:36:03,065 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/C of 47778d84645b4054cd2f50e381343edd into 54601378d8474ee08e4523c04f882635(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:03,065 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:03,065 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/C, priority=13, startTime=1731612962589; duration=0sec 2024-11-14T19:36:03,065 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:03,066 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:C 2024-11-14T19:36:03,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731612963095, max timestamp=1731612963110Num uniques 1054; 2024-11-14T19:36:03,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T19:36:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731612963095, max timestamp=1731612963110Num uniques 1054; 2024-11-14T19:36:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T19:36:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731612963095, max timestamp=1731612963110Num uniques 1054; 2024-11-14T19:36:03,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-14T19:36:03,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731612963170, max timestamp=1731612963183Num uniques 1013; 2024-11-14T19:36:03,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T19:36:03,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731612963170, max timestamp=1731612963183Num uniques 1013; 2024-11-14T19:36:03,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T19:36:03,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731612963170, max timestamp=1731612963183Num uniques 1013; 2024-11-14T19:36:03,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-14T19:36:03,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731612963227, max timestamp=1731612963254Num uniques 1022; 2024-11-14T19:36:03,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-14T19:36:03,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731612963227, max timestamp=1731612963254Num uniques 1022; 2024-11-14T19:36:03,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731612963227, max timestamp=1731612963254Num uniques 1022; 2024-11-14T19:36:03,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-14T19:36:03,258 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-14T19:36:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731612963299, max timestamp=1731612963324Num uniques 1031; 2024-11-14T19:36:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T19:36:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731612963299, max timestamp=1731612963324Num uniques 1031; 2024-11-14T19:36:03,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T19:36:03,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731612963299, max timestamp=1731612963324Num uniques 1031; 2024-11-14T19:36:03,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-14T19:36:03,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:03,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:03,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731612963370, max timestamp=1731612963425Num uniques 1040; 2024-11-14T19:36:03,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-14T19:36:03,429 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731612963370, max timestamp=1731612963425Num uniques 1040; 2024-11-14T19:36:03,430 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-14T19:36:03,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731612963370, max timestamp=1731612963425Num uniques 1040; 2024-11-14T19:36:03,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-14T19:36:03,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/3f03156399e647e78dbcecdd8f2fc3d2 2024-11-14T19:36:03,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8e8f881933314fbfadb4f1e88efb736f is 50, key is test_row_0/C:col10/1731612962595/Put/seqid=0 2024-11-14T19:36:03,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742092_1268 (size=12151) 2024-11-14T19:36:03,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:03,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731612963495, max timestamp=1731612963508Num uniques 1049; 2024-11-14T19:36:03,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-14T19:36:03,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731612963495, max timestamp=1731612963508Num uniques 1049; 2024-11-14T19:36:03,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731612963495, max timestamp=1731612963508Num uniques 1049; 2024-11-14T19:36:03,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-14T19:36:03,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-14T19:36:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731612963566, max timestamp=1731612963588Num uniques 1058; 2024-11-14T19:36:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-14T19:36:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731612963566, max timestamp=1731612963588Num uniques 1058; 2024-11-14T19:36:03,591 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1731612963566, max timestamp=1731612963588Num uniques 1058; 2024-11-14T19:36:03,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-14T19:36:03,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-11-14T19:36:03,625 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731612963607, max timestamp=1731612963622Num uniques 1017; 2024-11-14T19:36:03,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-14T19:36:03,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731612963607, max timestamp=1731612963622Num uniques 1017; 2024-11-14T19:36:03,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-14T19:36:03,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1731612963607, max timestamp=1731612963622Num uniques 1017; 2024-11-14T19:36:03,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-14T19:36:03,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:03,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:03,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731612963635, max timestamp=1731612963646Num uniques 1126; 2024-11-14T19:36:03,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=14926, suffix heap size=145072,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=326 2024-11-14T19:36:03,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731612963635, max timestamp=1731612963646Num uniques 1126; 2024-11-14T19:36:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=14926, suffix heap size=145072,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=326 2024-11-14T19:36:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=326, cellSize=14926, totalHeapSize=34144, min timestamp=1731612963635, max timestamp=1731612963646Num uniques 1126; 2024-11-14T19:36:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51566, new segment data size=14926, suffix heap size=145072,new segment heap size=34144 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1126, new segment cells count=326 2024-11-14T19:36:03,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613023668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613023668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:03,676 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731612963656, max timestamp=1731612963673Num uniques 1185; 2024-11-14T19:36:03,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=13050, suffix heap size=151208,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=285 2024-11-14T19:36:03,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731612963656, max timestamp=1731612963673Num uniques 1185; 2024-11-14T19:36:03,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=13050, suffix heap size=151208,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=285 2024-11-14T19:36:03,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731612963656, max timestamp=1731612963673Num uniques 1185; 2024-11-14T19:36:03,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54270, new segment data size=13050, suffix heap size=151208,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1185, new segment cells count=285 2024-11-14T19:36:03,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8864, totalHeapSize=20416, min timestamp=1731612963695, max timestamp=1731612963715Num uniques 1144; 2024-11-14T19:36:03,752 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52374, new segment data size=8864, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-14T19:36:03,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8864, totalHeapSize=20416, min timestamp=1731612963695, max timestamp=1731612963715Num uniques 1144; 2024-11-14T19:36:03,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52374, new segment data size=8864, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-14T19:36:03,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8868, totalHeapSize=20416, min timestamp=1731612963695, max timestamp=1731612963715Num uniques 1144; 2024-11-14T19:36:03,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52378, new segment data size=8868, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-11-14T19:36:03,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731612963763, max timestamp=1731612963781Num uniques 1053; 2024-11-14T19:36:03,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-14T19:36:03,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731612963763, max timestamp=1731612963781Num uniques 1053; 2024-11-14T19:36:03,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-14T19:36:03,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731612963763, max timestamp=1731612963781Num uniques 1053; 2024-11-14T19:36:03,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48225, new segment data size=9295, suffix heap size=137480,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1053, new segment cells count=203 2024-11-14T19:36:03,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:03,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:03,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:03,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1731612963795, max timestamp=1731612963807Num uniques 1062; 2024-11-14T19:36:03,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1731612963795, max timestamp=1731612963807Num uniques 1062; 2024-11-14T19:36:03,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=212, cellSize=9702, totalHeapSize=22288, min timestamp=1731612963795, max timestamp=1731612963807Num uniques 1062; 2024-11-14T19:36:03,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-11-14T19:36:03,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-11-14T19:36:03,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=9702, suffix heap size=138416,new segment heap size=22288 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=212 2024-11-14T19:36:03,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8e8f881933314fbfadb4f1e88efb736f 2024-11-14T19:36:03,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731612963856, max timestamp=1731612963861Num uniques 1071; 2024-11-14T19:36:03,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731612963856, max timestamp=1731612963861Num uniques 1071; 2024-11-14T19:36:03,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-14T19:36:03,866 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=10116, suffix heap size=139352,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=221 2024-11-14T19:36:03,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/58864f773e3e40ef942449041a26df70 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/58864f773e3e40ef942449041a26df70 2024-11-14T19:36:03,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731612963856, max timestamp=1731612963861Num uniques 1071; 2024-11-14T19:36:03,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49046, new segment data size=6870, suffix heap size=139352,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1071, new segment cells count=150 2024-11-14T19:36:03,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/58864f773e3e40ef942449041a26df70, entries=150, sequenceid=182, filesize=11.9 K 2024-11-14T19:36:03,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/3f03156399e647e78dbcecdd8f2fc3d2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3f03156399e647e78dbcecdd8f2fc3d2 2024-11-14T19:36:03,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3f03156399e647e78dbcecdd8f2fc3d2, entries=150, sequenceid=182, filesize=11.9 K 2024-11-14T19:36:03,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/8e8f881933314fbfadb4f1e88efb736f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8e8f881933314fbfadb4f1e88efb736f 2024-11-14T19:36:03,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8e8f881933314fbfadb4f1e88efb736f, entries=150, sequenceid=182, filesize=11.9 K 2024-11-14T19:36:03,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~65.34 KB/66912, currentSize=97.46 KB/99804 for 47778d84645b4054cd2f50e381343edd in 1287ms, sequenceid=182, compaction requested=false 2024-11-14T19:36:03,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:03,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=104.17 KB heapSize=292.27 KB 2024-11-14T19:36:03,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:03,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:03,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:03,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:03,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:03,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:03,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/7b887b36b15145a4aa22a2c96cac08bc is 50, key is test_row_0/A:col10/1731612963884/Put/seqid=0 2024-11-14T19:36:03,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742093_1269 (size=12301) 2024-11-14T19:36:03,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613023900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:03,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613023901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:03,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:03,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613023902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613023904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613023905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:03,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:03,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613024013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613024013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613024013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613024013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613024013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:04,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:04,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613024223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613024223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613024223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613024223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613024223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:04,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:04,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/7b887b36b15145a4aa22a2c96cac08bc 2024-11-14T19:36:04,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/b8eb601113ce47eca63c507c42d9d052 is 50, key is test_row_0/B:col10/1731612963884/Put/seqid=0 2024-11-14T19:36:04,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:04,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:04,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742094_1270 (size=12301) 2024-11-14T19:36:04,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613024533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613024533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613024533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613024533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613024533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:04,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:04,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:04,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:04,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/b8eb601113ce47eca63c507c42d9d052 2024-11-14T19:36:04,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/cce0fb717f9d46d2b01bda48d3e273e1 is 50, key is test_row_0/C:col10/1731612963884/Put/seqid=0 2024-11-14T19:36:04,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742095_1271 (size=12301) 2024-11-14T19:36:04,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:04,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:04,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:04,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:04,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:05,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:05,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:05,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:05,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:05,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613025043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:05,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613025044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613025044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613025044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:05,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613025044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:05,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:05,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:05,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:05,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:05,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:05,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.85 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/cce0fb717f9d46d2b01bda48d3e273e1 2024-11-14T19:36:05,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/7b887b36b15145a4aa22a2c96cac08bc as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/7b887b36b15145a4aa22a2c96cac08bc 2024-11-14T19:36:05,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/7b887b36b15145a4aa22a2c96cac08bc, entries=150, sequenceid=527, filesize=12.0 K 2024-11-14T19:36:05,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/b8eb601113ce47eca63c507c42d9d052 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/b8eb601113ce47eca63c507c42d9d052 2024-11-14T19:36:05,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/b8eb601113ce47eca63c507c42d9d052, entries=150, sequenceid=527, filesize=12.0 K 2024-11-14T19:36:05,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/cce0fb717f9d46d2b01bda48d3e273e1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/cce0fb717f9d46d2b01bda48d3e273e1 2024-11-14T19:36:05,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/cce0fb717f9d46d2b01bda48d3e273e1, entries=150, sequenceid=527, filesize=12.0 K 2024-11-14T19:36:05,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~110.88 KB/113544, heapSize ~312.15 KB/319640, currentSize=67.09 KB/68700 for 47778d84645b4054cd2f50e381343edd in 1350ms, sequenceid=527, compaction requested=true 2024-11-14T19:36:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:05,238 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:05,238 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:05,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:05,239 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:05,239 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:05,239 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/A is initiating minor compaction (all files) 2024-11-14T19:36:05,239 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/B is initiating minor compaction (all files) 2024-11-14T19:36:05,239 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/B in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,239 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/A in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,239 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/451e98dd6bd047a4967d67bfcda87238, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/58864f773e3e40ef942449041a26df70, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/7b887b36b15145a4aa22a2c96cac08bc] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=36.0 K 2024-11-14T19:36:05,239 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/4055822feaf14deb80d77f321bbc2339, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3f03156399e647e78dbcecdd8f2fc3d2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/b8eb601113ce47eca63c507c42d9d052] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=36.0 K 2024-11-14T19:36:05,240 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4055822feaf14deb80d77f321bbc2339, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731612961288 2024-11-14T19:36:05,240 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 451e98dd6bd047a4967d67bfcda87238, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731612961288 2024-11-14T19:36:05,240 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f03156399e647e78dbcecdd8f2fc3d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731612961381 2024-11-14T19:36:05,240 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58864f773e3e40ef942449041a26df70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731612961381 2024-11-14T19:36:05,240 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b8eb601113ce47eca63c507c42d9d052, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731612963882 2024-11-14T19:36:05,240 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b887b36b15145a4aa22a2c96cac08bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731612963882 2024-11-14T19:36:05,247 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:05,247 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#A#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:05,247 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/82a6cb34d01f4d65a807160b334e8e84 is 50, key is test_row_0/B:col10/1731612963884/Put/seqid=0 2024-11-14T19:36:05,247 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/e4f061f9beb14131963ee617a66875ca is 50, key is test_row_0/A:col10/1731612963884/Put/seqid=0 2024-11-14T19:36:05,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742096_1272 (size=12711) 2024-11-14T19:36:05,263 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/82a6cb34d01f4d65a807160b334e8e84 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82a6cb34d01f4d65a807160b334e8e84 2024-11-14T19:36:05,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742097_1273 (size=12711) 2024-11-14T19:36:05,268 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/B of 47778d84645b4054cd2f50e381343edd into 82a6cb34d01f4d65a807160b334e8e84(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:05,268 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:05,268 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/B, priority=13, startTime=1731612965238; duration=0sec 2024-11-14T19:36:05,269 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:05,269 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:B 2024-11-14T19:36:05,269 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:05,270 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:05,270 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/C is initiating minor compaction (all files) 2024-11-14T19:36:05,270 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/C in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,270 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/54601378d8474ee08e4523c04f882635, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8e8f881933314fbfadb4f1e88efb736f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/cce0fb717f9d46d2b01bda48d3e273e1] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=36.0 K 2024-11-14T19:36:05,270 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 54601378d8474ee08e4523c04f882635, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731612961288 2024-11-14T19:36:05,270 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e8f881933314fbfadb4f1e88efb736f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731612961381 2024-11-14T19:36:05,271 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cce0fb717f9d46d2b01bda48d3e273e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731612963882 2024-11-14T19:36:05,279 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:05,279 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/879d14721b774e5dbd919fdeb9f2d68a is 50, key is test_row_0/C:col10/1731612963884/Put/seqid=0 2024-11-14T19:36:05,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742098_1274 (size=12711) 2024-11-14T19:36:05,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-14T19:36:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:05,318 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:36:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:05,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/377e689ec0344446ab9c4f9320b93d4e is 50, key is test_row_0/A:col10/1731612963900/Put/seqid=0 2024-11-14T19:36:05,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742099_1275 (size=12301) 2024-11-14T19:36:05,670 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/e4f061f9beb14131963ee617a66875ca as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e4f061f9beb14131963ee617a66875ca 2024-11-14T19:36:05,676 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/A of 47778d84645b4054cd2f50e381343edd into e4f061f9beb14131963ee617a66875ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:05,676 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:05,676 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/A, priority=13, startTime=1731612965238; duration=0sec 2024-11-14T19:36:05,676 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:05,676 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:A 2024-11-14T19:36:05,692 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/879d14721b774e5dbd919fdeb9f2d68a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/879d14721b774e5dbd919fdeb9f2d68a 2024-11-14T19:36:05,697 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/C of 47778d84645b4054cd2f50e381343edd into 879d14721b774e5dbd919fdeb9f2d68a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:05,697 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:05,697 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/C, priority=13, startTime=1731612965238; duration=0sec 2024-11-14T19:36:05,697 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:05,697 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:C 2024-11-14T19:36:05,734 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/377e689ec0344446ab9c4f9320b93d4e 2024-11-14T19:36:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/867004c3ed33491aa1c2028ac26f830d is 50, key is test_row_0/B:col10/1731612963900/Put/seqid=0 2024-11-14T19:36:05,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742100_1276 (size=12301) 2024-11-14T19:36:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:06,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:06,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613026069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613026070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613026071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613026075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613026076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,149 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/867004c3ed33491aa1c2028ac26f830d 2024-11-14T19:36:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1d1a9cc0d70745be98864bb383ea575d is 50, key is test_row_0/C:col10/1731612963900/Put/seqid=0 2024-11-14T19:36:06,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742101_1277 (size=12301) 2024-11-14T19:36:06,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613026183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613026183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613026183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613026183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613026183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613026393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 305 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613026393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613026393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613026393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613026393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,569 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1d1a9cc0d70745be98864bb383ea575d 2024-11-14T19:36:06,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/377e689ec0344446ab9c4f9320b93d4e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/377e689ec0344446ab9c4f9320b93d4e 2024-11-14T19:36:06,578 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/377e689ec0344446ab9c4f9320b93d4e, entries=150, sequenceid=541, filesize=12.0 K 2024-11-14T19:36:06,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/867004c3ed33491aa1c2028ac26f830d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/867004c3ed33491aa1c2028ac26f830d 2024-11-14T19:36:06,583 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/867004c3ed33491aa1c2028ac26f830d, entries=150, sequenceid=541, filesize=12.0 K 2024-11-14T19:36:06,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1d1a9cc0d70745be98864bb383ea575d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1d1a9cc0d70745be98864bb383ea575d 2024-11-14T19:36:06,588 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1d1a9cc0d70745be98864bb383ea575d, entries=150, sequenceid=541, filesize=12.0 K 2024-11-14T19:36:06,588 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 47778d84645b4054cd2f50e381343edd in 1270ms, sequenceid=541, compaction requested=false 2024-11-14T19:36:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-14T19:36:06,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-14T19:36:06,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-14T19:36:06,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.9430 sec 2024-11-14T19:36:06,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 4.9480 sec 2024-11-14T19:36:06,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:06,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:36:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:06,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:06,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/e8eb2268aa1144a08f92b9174a025e06 is 50, key is test_row_0/A:col10/1731612966070/Put/seqid=0 2024-11-14T19:36:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742102_1278 (size=12301) 2024-11-14T19:36:06,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613026714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613026714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613026715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613026715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613026716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613026823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613026823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 309 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613026823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613026823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:06,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:06,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613026833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:06,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613027033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613027033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613027033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613027033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613027043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/e8eb2268aa1144a08f92b9174a025e06 2024-11-14T19:36:07,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/1fd0b5ce93d2452fab2393c0c4ae4071 is 50, key is test_row_0/B:col10/1731612966070/Put/seqid=0 2024-11-14T19:36:07,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742103_1279 (size=12301) 2024-11-14T19:36:07,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613027343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613027344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 311 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613027344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613027344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613027353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/1fd0b5ce93d2452fab2393c0c4ae4071 2024-11-14T19:36:07,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/692384462f3642a2b70d91071687bbbf is 50, key is test_row_0/C:col10/1731612966070/Put/seqid=0 2024-11-14T19:36:07,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742104_1280 (size=12301) 2024-11-14T19:36:07,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613027852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613027853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613027853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613027853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613027863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:07,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=564 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/692384462f3642a2b70d91071687bbbf 2024-11-14T19:36:07,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/e8eb2268aa1144a08f92b9174a025e06 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e8eb2268aa1144a08f92b9174a025e06 2024-11-14T19:36:07,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e8eb2268aa1144a08f92b9174a025e06, entries=150, sequenceid=564, filesize=12.0 K 2024-11-14T19:36:07,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/1fd0b5ce93d2452fab2393c0c4ae4071 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/1fd0b5ce93d2452fab2393c0c4ae4071 2024-11-14T19:36:07,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/1fd0b5ce93d2452fab2393c0c4ae4071, entries=150, sequenceid=564, filesize=12.0 K 2024-11-14T19:36:07,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/692384462f3642a2b70d91071687bbbf as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/692384462f3642a2b70d91071687bbbf 2024-11-14T19:36:07,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/692384462f3642a2b70d91071687bbbf, entries=150, sequenceid=564, filesize=12.0 K 2024-11-14T19:36:07,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 47778d84645b4054cd2f50e381343edd in 1254ms, sequenceid=564, compaction requested=true 2024-11-14T19:36:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:07,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:07,961 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:07,961 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:07,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:07,962 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:07,962 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/B is initiating minor compaction (all files) 2024-11-14T19:36:07,962 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/B in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:07,962 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82a6cb34d01f4d65a807160b334e8e84, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/867004c3ed33491aa1c2028ac26f830d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/1fd0b5ce93d2452fab2393c0c4ae4071] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=36.4 K 2024-11-14T19:36:07,962 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:07,962 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/A is initiating minor compaction (all files) 2024-11-14T19:36:07,962 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/A in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:07,962 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e4f061f9beb14131963ee617a66875ca, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/377e689ec0344446ab9c4f9320b93d4e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e8eb2268aa1144a08f92b9174a025e06] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=36.4 K 2024-11-14T19:36:07,963 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 82a6cb34d01f4d65a807160b334e8e84, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731612963882 2024-11-14T19:36:07,963 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4f061f9beb14131963ee617a66875ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731612963882 2024-11-14T19:36:07,963 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 867004c3ed33491aa1c2028ac26f830d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731612963900 2024-11-14T19:36:07,963 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 377e689ec0344446ab9c4f9320b93d4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731612963900 2024-11-14T19:36:07,964 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fd0b5ce93d2452fab2393c0c4ae4071, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731612966070 2024-11-14T19:36:07,964 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8eb2268aa1144a08f92b9174a025e06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731612966070 2024-11-14T19:36:07,977 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#A#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:07,978 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/05a7e69baf87429d88b77f992a7415ff is 50, key is test_row_0/A:col10/1731612966070/Put/seqid=0 2024-11-14T19:36:07,980 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:07,980 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/82807a9e681941199c166531f46e3481 is 50, key is test_row_0/B:col10/1731612966070/Put/seqid=0 2024-11-14T19:36:08,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742105_1281 (size=12813) 2024-11-14T19:36:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742106_1282 (size=12813) 2024-11-14T19:36:08,007 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/82807a9e681941199c166531f46e3481 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82807a9e681941199c166531f46e3481 2024-11-14T19:36:08,013 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/B of 47778d84645b4054cd2f50e381343edd into 82807a9e681941199c166531f46e3481(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:08,013 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:08,013 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/B, priority=13, startTime=1731612967961; duration=0sec 2024-11-14T19:36:08,013 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:08,013 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:B 2024-11-14T19:36:08,013 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:08,015 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:08,015 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/C is initiating minor compaction (all files) 2024-11-14T19:36:08,015 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/C in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:08,015 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/879d14721b774e5dbd919fdeb9f2d68a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1d1a9cc0d70745be98864bb383ea575d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/692384462f3642a2b70d91071687bbbf] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=36.4 K 2024-11-14T19:36:08,015 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 879d14721b774e5dbd919fdeb9f2d68a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731612963882 2024-11-14T19:36:08,016 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d1a9cc0d70745be98864bb383ea575d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1731612963900 2024-11-14T19:36:08,016 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 692384462f3642a2b70d91071687bbbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731612966070 2024-11-14T19:36:08,024 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:08,025 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/d7c7a8604bda4ff4a6c6b41403d88b75 is 50, key is test_row_0/C:col10/1731612966070/Put/seqid=0 2024-11-14T19:36:08,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742107_1283 (size=12813) 2024-11-14T19:36:08,413 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/05a7e69baf87429d88b77f992a7415ff as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/05a7e69baf87429d88b77f992a7415ff 2024-11-14T19:36:08,419 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/A of 47778d84645b4054cd2f50e381343edd into 05a7e69baf87429d88b77f992a7415ff(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:08,419 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:08,419 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/A, priority=13, startTime=1731612967960; duration=0sec 2024-11-14T19:36:08,419 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:08,419 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:A 2024-11-14T19:36:08,451 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/d7c7a8604bda4ff4a6c6b41403d88b75 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/d7c7a8604bda4ff4a6c6b41403d88b75 2024-11-14T19:36:08,458 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 47778d84645b4054cd2f50e381343edd/C of 47778d84645b4054cd2f50e381343edd into d7c7a8604bda4ff4a6c6b41403d88b75(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:08,458 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:08,458 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/C, priority=13, startTime=1731612967961; duration=0sec 2024-11-14T19:36:08,458 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:08,458 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:C 2024-11-14T19:36:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:08,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:36:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:08,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:08,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/fa72c751ebe94bfeae64b367a4d05845 is 50, key is test_row_0/A:col10/1731612966713/Put/seqid=0 2024-11-14T19:36:08,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742108_1284 (size=17181) 2024-11-14T19:36:08,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613028884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613028885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:08,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613028887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:08,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613028888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613028888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:08,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613028993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:08,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613028994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:08,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613029003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613029003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613029203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613029204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613029213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613029213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/fa72c751ebe94bfeae64b367a4d05845 2024-11-14T19:36:09,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/0fa5fb80d519441e90122f8fc14d6dc4 is 50, key is test_row_0/B:col10/1731612966713/Put/seqid=0 2024-11-14T19:36:09,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742109_1285 (size=12301) 2024-11-14T19:36:09,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613029513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613029513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613029513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613029523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613029523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:09,526 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:09,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/0fa5fb80d519441e90122f8fc14d6dc4 2024-11-14T19:36:09,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/a94fe85deee44265add6e73639055ec1 is 50, key is test_row_0/C:col10/1731612966713/Put/seqid=0 2024-11-14T19:36:09,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742110_1286 (size=12301) 2024-11-14T19:36:09,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=581 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/a94fe85deee44265add6e73639055ec1 2024-11-14T19:36:09,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/fa72c751ebe94bfeae64b367a4d05845 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/fa72c751ebe94bfeae64b367a4d05845 2024-11-14T19:36:09,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/fa72c751ebe94bfeae64b367a4d05845, entries=250, sequenceid=581, filesize=16.8 K 2024-11-14T19:36:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/0fa5fb80d519441e90122f8fc14d6dc4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/0fa5fb80d519441e90122f8fc14d6dc4 2024-11-14T19:36:09,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/0fa5fb80d519441e90122f8fc14d6dc4, entries=150, sequenceid=581, filesize=12.0 K 2024-11-14T19:36:09,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/a94fe85deee44265add6e73639055ec1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/a94fe85deee44265add6e73639055ec1 2024-11-14T19:36:09,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/a94fe85deee44265add6e73639055ec1, entries=150, sequenceid=581, filesize=12.0 K 2024-11-14T19:36:09,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 47778d84645b4054cd2f50e381343edd in 869ms, sequenceid=581, compaction requested=false 2024-11-14T19:36:09,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:09,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-14T19:36:09,843 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:09,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-14T19:36:09,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T19:36:09,846 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:09,846 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:09,846 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T19:36:09,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-14T19:36:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:09,998 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:09,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:10,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/ba0a68e2e66a4b91b356c440f0616bae is 50, key is test_row_0/A:col10/1731612968886/Put/seqid=0 2024-11-14T19:36:10,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742111_1287 (size=12301) 2024-11-14T19:36:10,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:10,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:10,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613030034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613030035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613030035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613030036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613030037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613030143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613030143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613030143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613030143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613030143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T19:36:10,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613030353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613030353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613030353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613030353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613030354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,407 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/ba0a68e2e66a4b91b356c440f0616bae 2024-11-14T19:36:10,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/a37e6a5e56574ad2bcf5b92e892bb1c7 is 50, key is test_row_0/B:col10/1731612968886/Put/seqid=0 2024-11-14T19:36:10,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742112_1288 (size=12301) 2024-11-14T19:36:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T19:36:10,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613030663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613030663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613030663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613030663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:10,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613030663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:10,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:10,819 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/a37e6a5e56574ad2bcf5b92e892bb1c7 2024-11-14T19:36:10,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1adc4d1b2eff4a00ac6f3bdc61da9452 is 50, key is test_row_0/C:col10/1731612968886/Put/seqid=0 2024-11-14T19:36:10,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742113_1289 (size=12301) 2024-11-14T19:36:10,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T19:36:11,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613031173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613031173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 329 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613031173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:11,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613031173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:11,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613031174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:11,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:11,229 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1adc4d1b2eff4a00ac6f3bdc61da9452 2024-11-14T19:36:11,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/ba0a68e2e66a4b91b356c440f0616bae as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ba0a68e2e66a4b91b356c440f0616bae 2024-11-14T19:36:11,237 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ba0a68e2e66a4b91b356c440f0616bae, entries=150, sequenceid=599, filesize=12.0 K 2024-11-14T19:36:11,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/a37e6a5e56574ad2bcf5b92e892bb1c7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/a37e6a5e56574ad2bcf5b92e892bb1c7 2024-11-14T19:36:11,242 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/a37e6a5e56574ad2bcf5b92e892bb1c7, entries=150, sequenceid=599, filesize=12.0 K 2024-11-14T19:36:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/1adc4d1b2eff4a00ac6f3bdc61da9452 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1adc4d1b2eff4a00ac6f3bdc61da9452 2024-11-14T19:36:11,246 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1adc4d1b2eff4a00ac6f3bdc61da9452, entries=150, sequenceid=599, filesize=12.0 K 2024-11-14T19:36:11,247 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 47778d84645b4054cd2f50e381343edd in 1248ms, sequenceid=599, compaction requested=true 2024-11-14T19:36:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-14T19:36:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-14T19:36:11,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-14T19:36:11,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-11-14T19:36:11,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4070 sec 2024-11-14T19:36:11,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-14T19:36:11,994 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:11,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-14T19:36:11,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T19:36:11,996 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:11,996 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:11,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T19:36:12,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-14T19:36:12,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:12,149 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T19:36:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:12,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:12,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/ebb13d5b32c24ffd9aec1bd43cc439a8 is 50, key is test_row_0/A:col10/1731612970036/Put/seqid=0 2024-11-14T19:36:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742114_1290 (size=12301) 2024-11-14T19:36:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:12,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:12,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613032199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613032201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613032204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613032205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613032205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T19:36:12,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613032313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613032313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613032313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613032313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613032313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613032523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613032523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613032523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613032523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613032523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,556 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/ebb13d5b32c24ffd9aec1bd43cc439a8 2024-11-14T19:36:12,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/c43d701dfa474c3d9b6f43b2c27d1a63 is 50, key is test_row_0/B:col10/1731612970036/Put/seqid=0 2024-11-14T19:36:12,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742115_1291 (size=12301) 2024-11-14T19:36:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T19:36:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613032833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613032833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613032834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613032834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613032835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:12,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:12,968 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/c43d701dfa474c3d9b6f43b2c27d1a63 2024-11-14T19:36:13,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/f080426ede9546eea07f6d7b811ade4c is 50, key is test_row_0/C:col10/1731612970036/Put/seqid=0 2024-11-14T19:36:13,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742116_1292 (size=12301) 2024-11-14T19:36:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T19:36:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613033344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613033344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613033344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:13,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613033344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:13,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613033353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:13,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:13,452 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=613 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/f080426ede9546eea07f6d7b811ade4c 2024-11-14T19:36:13,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/ebb13d5b32c24ffd9aec1bd43cc439a8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ebb13d5b32c24ffd9aec1bd43cc439a8 2024-11-14T19:36:13,474 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ebb13d5b32c24ffd9aec1bd43cc439a8, entries=150, sequenceid=613, filesize=12.0 K 2024-11-14T19:36:13,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/c43d701dfa474c3d9b6f43b2c27d1a63 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/c43d701dfa474c3d9b6f43b2c27d1a63 2024-11-14T19:36:13,481 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/c43d701dfa474c3d9b6f43b2c27d1a63, entries=150, sequenceid=613, filesize=12.0 K 2024-11-14T19:36:13,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/f080426ede9546eea07f6d7b811ade4c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f080426ede9546eea07f6d7b811ade4c 2024-11-14T19:36:13,490 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f080426ede9546eea07f6d7b811ade4c, entries=150, sequenceid=613, filesize=12.0 K 2024-11-14T19:36:13,491 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 47778d84645b4054cd2f50e381343edd in 1343ms, sequenceid=613, compaction requested=true 2024-11-14T19:36:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:13,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-14T19:36:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-14T19:36:13,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-14T19:36:13,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-14T19:36:13,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5030 sec 2024-11-14T19:36:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-14T19:36:14,144 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:14,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-14T19:36:14,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T19:36:14,147 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:14,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T19:36:14,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-14T19:36:14,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:14,302 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:14,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:14,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/06dab6d8cdcf42c88339807055494941 is 50, key is test_row_0/A:col10/1731612972204/Put/seqid=0 2024-11-14T19:36:14,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742117_1293 (size=12301) 2024-11-14T19:36:14,321 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/06dab6d8cdcf42c88339807055494941 2024-11-14T19:36:14,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/7c7fe466ad4f4fe781c8de74b1b0dc8f is 50, key is test_row_0/B:col10/1731612972204/Put/seqid=0 2024-11-14T19:36:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:14,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. as already flushing 2024-11-14T19:36:14,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613034384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742118_1294 (size=12301) 2024-11-14T19:36:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 339 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613034386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613034386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613034387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613034391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T19:36:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613034493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 340 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613034503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613034504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613034504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613034504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613034705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 341 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613034713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613034714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613034714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613034714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:14,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T19:36:14,793 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/7c7fe466ad4f4fe781c8de74b1b0dc8f 2024-11-14T19:36:14,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/b8ad29a612cf47b484ef71ff2aa294ad is 50, key is test_row_0/C:col10/1731612972204/Put/seqid=0 2024-11-14T19:36:14,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742119_1295 (size=12301) 2024-11-14T19:36:15,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47068 deadline: 1731613035023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:15,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47086 deadline: 1731613035023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:15,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47118 deadline: 1731613035033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:15,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47096 deadline: 1731613035033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:15,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47090 deadline: 1731613035033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:15,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:15,250 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=631 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/b8ad29a612cf47b484ef71ff2aa294ad 2024-11-14T19:36:15,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/06dab6d8cdcf42c88339807055494941 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/06dab6d8cdcf42c88339807055494941 2024-11-14T19:36:15,260 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/06dab6d8cdcf42c88339807055494941, entries=150, sequenceid=631, filesize=12.0 K 2024-11-14T19:36:15,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/7c7fe466ad4f4fe781c8de74b1b0dc8f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/7c7fe466ad4f4fe781c8de74b1b0dc8f 2024-11-14T19:36:15,267 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/7c7fe466ad4f4fe781c8de74b1b0dc8f, entries=150, sequenceid=631, filesize=12.0 K 2024-11-14T19:36:15,268 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-14T19:36:15,268 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,268 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,268 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/b8ad29a612cf47b484ef71ff2aa294ad as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8ad29a612cf47b484ef71ff2aa294ad 2024-11-14T19:36:15,276 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-14T19:36:15,276 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,276 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,276 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,278 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-14T19:36:15,278 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,278 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,278 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,279 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-14T19:36:15,279 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,279 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,279 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,281 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8ad29a612cf47b484ef71ff2aa294ad, entries=150, sequenceid=631, filesize=12.0 K 2024-11-14T19:36:15,282 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-14T19:36:15,282 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,282 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,282 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,283 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 47778d84645b4054cd2f50e381343edd in 980ms, sequenceid=631, compaction requested=true 2024-11-14T19:36:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T19:36:15,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:15,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:15,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-14T19:36:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-14T19:36:15,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-14T19:36:15,286 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1370 sec 2024-11-14T19:36:15,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.1430 sec 2024-11-14T19:36:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:15,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:36:15,534 INFO [Thread-1098 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1098. 2024-11-14T19:36:15,535 DEBUG [Thread-1098 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,535 INFO [Thread-1102 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1102. 2024-11-14T19:36:15,535 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,535 DEBUG [Thread-1102 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:15,535 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,535 DEBUG [Thread-1098 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:15,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,535 DEBUG [Thread-1102 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:15,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:15,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:15,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/022dfd7032564b0aa706049327af1790 is 50, key is test_row_0/A:col10/1731612975533/Put/seqid=0 2024-11-14T19:36:15,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742120_1296 (size=12301) 2024-11-14T19:36:15,545 INFO [Thread-1104 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1104. 2024-11-14T19:36:15,545 DEBUG [Thread-1104 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,545 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,545 DEBUG [Thread-1104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,555 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-14T19:36:15,555 INFO [Thread-1100 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1100. 2024-11-14T19:36:15,555 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,555 DEBUG [Thread-1100 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:15,555 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,555 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,556 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,556 DEBUG [Thread-1100 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:15,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:15,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/022dfd7032564b0aa706049327af1790 2024-11-14T19:36:15,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/26ebd0c9643a4b59935f8c8a6c8822fd is 50, key is test_row_0/B:col10/1731612975533/Put/seqid=0 2024-11-14T19:36:15,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742121_1297 (size=12301) 2024-11-14T19:36:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-14T19:36:16,294 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 288 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2895 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2908 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2995 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2925 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2915 2024-11-14T19:36:16,294 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T19:36:16,294 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:36:16,294 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:36:16,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:16,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:16,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:16,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T19:36:16,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:16,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T19:36:16,298 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612976298"}]},"ts":"1731612976298"} 2024-11-14T19:36:16,299 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T19:36:16,299 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T19:36:16,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:36:16,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, UNASSIGN}] 2024-11-14T19:36:16,303 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, UNASSIGN 2024-11-14T19:36:16,303 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=47778d84645b4054cd2f50e381343edd, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:16,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, UNASSIGN because future has completed 2024-11-14T19:36:16,305 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:36:16,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:36:16,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/26ebd0c9643a4b59935f8c8a6c8822fd 2024-11-14T19:36:16,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/6ed9cb62da52491d89cf30ccb79b0c01 is 50, key is test_row_0/C:col10/1731612975533/Put/seqid=0 2024-11-14T19:36:16,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742122_1298 (size=12301) 2024-11-14T19:36:16,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/6ed9cb62da52491d89cf30ccb79b0c01 2024-11-14T19:36:16,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/022dfd7032564b0aa706049327af1790 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/022dfd7032564b0aa706049327af1790 2024-11-14T19:36:16,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/022dfd7032564b0aa706049327af1790, entries=150, sequenceid=647, filesize=12.0 K 2024-11-14T19:36:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/26ebd0c9643a4b59935f8c8a6c8822fd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/26ebd0c9643a4b59935f8c8a6c8822fd 2024-11-14T19:36:16,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/26ebd0c9643a4b59935f8c8a6c8822fd, entries=150, sequenceid=647, filesize=12.0 K 2024-11-14T19:36:16,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/6ed9cb62da52491d89cf30ccb79b0c01 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6ed9cb62da52491d89cf30ccb79b0c01 2024-11-14T19:36:16,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6ed9cb62da52491d89cf30ccb79b0c01, entries=150, sequenceid=647, filesize=12.0 K 2024-11-14T19:36:16,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for 47778d84645b4054cd2f50e381343edd in 861ms, sequenceid=647, compaction requested=true 2024-11-14T19:36:16,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:16,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:16,395 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T19:36:16,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:16,395 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-14T19:36:16,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:16,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:16,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 47778d84645b4054cd2f50e381343edd:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:16,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74318 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 79198 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/A is initiating minor compaction (all files) 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 47778d84645b4054cd2f50e381343edd/B is initiating minor compaction (all files) 2024-11-14T19:36:16,398 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/B in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:16,398 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 47778d84645b4054cd2f50e381343edd/A in TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:16,398 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82807a9e681941199c166531f46e3481, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/0fa5fb80d519441e90122f8fc14d6dc4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/a37e6a5e56574ad2bcf5b92e892bb1c7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/c43d701dfa474c3d9b6f43b2c27d1a63, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/7c7fe466ad4f4fe781c8de74b1b0dc8f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/26ebd0c9643a4b59935f8c8a6c8822fd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=72.6 K 2024-11-14T19:36:16,398 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/05a7e69baf87429d88b77f992a7415ff, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/fa72c751ebe94bfeae64b367a4d05845, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ba0a68e2e66a4b91b356c440f0616bae, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ebb13d5b32c24ffd9aec1bd43cc439a8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/06dab6d8cdcf42c88339807055494941, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/022dfd7032564b0aa706049327af1790] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp, totalSize=77.3 K 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 82807a9e681941199c166531f46e3481, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731612966070 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05a7e69baf87429d88b77f992a7415ff, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=564, earliestPutTs=1731612966070 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fa5fb80d519441e90122f8fc14d6dc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1731612966713 2024-11-14T19:36:16,398 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa72c751ebe94bfeae64b367a4d05845, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=581, earliestPutTs=1731612966713 2024-11-14T19:36:16,399 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a37e6a5e56574ad2bcf5b92e892bb1c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731612968886 2024-11-14T19:36:16,399 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0a68e2e66a4b91b356c440f0616bae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731612968886 2024-11-14T19:36:16,399 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c43d701dfa474c3d9b6f43b2c27d1a63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731612970032 2024-11-14T19:36:16,399 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebb13d5b32c24ffd9aec1bd43cc439a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=613, earliestPutTs=1731612970032 2024-11-14T19:36:16,399 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c7fe466ad4f4fe781c8de74b1b0dc8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731612972200 2024-11-14T19:36:16,400 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06dab6d8cdcf42c88339807055494941, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=631, earliestPutTs=1731612972200 2024-11-14T19:36:16,400 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 26ebd0c9643a4b59935f8c8a6c8822fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731612974385 2024-11-14T19:36:16,400 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 022dfd7032564b0aa706049327af1790, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1731612974385 2024-11-14T19:36:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T19:36:16,417 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:16,418 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/e51a849b69cc4d02b2ac4545ca31397a is 50, key is test_row_0/B:col10/1731612975533/Put/seqid=0 2024-11-14T19:36:16,430 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 47778d84645b4054cd2f50e381343edd#A#compaction#253 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:16,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742123_1299 (size=13017) 2024-11-14T19:36:16,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/1af015ec3d5f4d23babf9212f02ab4f7 is 50, key is test_row_0/A:col10/1731612975533/Put/seqid=0 2024-11-14T19:36:16,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742124_1300 (size=13017) 2024-11-14T19:36:16,458 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:16,458 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:36:16,458 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing 47778d84645b4054cd2f50e381343edd, disabling compactions & flushes 2024-11-14T19:36:16,458 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T19:36:16,837 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/e51a849b69cc4d02b2ac4545ca31397a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/e51a849b69cc4d02b2ac4545ca31397a 2024-11-14T19:36:16,845 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/1af015ec3d5f4d23babf9212f02ab4f7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1af015ec3d5f4d23babf9212f02ab4f7 2024-11-14T19:36:16,845 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 47778d84645b4054cd2f50e381343edd/B of 47778d84645b4054cd2f50e381343edd into e51a849b69cc4d02b2ac4545ca31397a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:16,846 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:16,846 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/B, priority=10, startTime=1731612976395; duration=0sec 2024-11-14T19:36:16,846 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:16,846 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:B 2024-11-14T19:36:16,846 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. because compaction request was cancelled 2024-11-14T19:36:16,846 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:C 2024-11-14T19:36:16,851 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 47778d84645b4054cd2f50e381343edd/A of 47778d84645b4054cd2f50e381343edd into 1af015ec3d5f4d23babf9212f02ab4f7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:16,851 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 47778d84645b4054cd2f50e381343edd: 2024-11-14T19:36:16,851 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., storeName=47778d84645b4054cd2f50e381343edd/A, priority=10, startTime=1731612976395; duration=0sec 2024-11-14T19:36:16,851 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:16,851 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:16,851 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:16,851 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 47778d84645b4054cd2f50e381343edd:A 2024-11-14T19:36:16,851 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. after waiting 0 ms 2024-11-14T19:36:16,851 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:16,851 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(2902): Flushing 47778d84645b4054cd2f50e381343edd 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T19:36:16,852 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=A 2024-11-14T19:36:16,852 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:16,852 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=B 2024-11-14T19:36:16,852 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:16,852 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 47778d84645b4054cd2f50e381343edd, store=C 2024-11-14T19:36:16,852 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:16,856 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/eb79ed134f3a4e538e68119b35915015 is 50, key is test_row_0/A:col10/1731612975554/Put/seqid=0 2024-11-14T19:36:16,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742125_1301 (size=12301) 2024-11-14T19:36:16,860 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T19:36:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T19:36:17,260 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/eb79ed134f3a4e538e68119b35915015 2024-11-14T19:36:17,267 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/990efb78c1ed48dda153b21e495bcae3 is 50, key is test_row_0/B:col10/1731612975554/Put/seqid=0 2024-11-14T19:36:17,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742126_1302 (size=12301) 2024-11-14T19:36:17,278 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/990efb78c1ed48dda153b21e495bcae3 2024-11-14T19:36:17,287 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/10769ff93be34d91bc28b171096968c8 is 50, key is test_row_0/C:col10/1731612975554/Put/seqid=0 2024-11-14T19:36:17,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742127_1303 (size=12301) 2024-11-14T19:36:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T19:36:17,693 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/10769ff93be34d91bc28b171096968c8 2024-11-14T19:36:17,700 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/A/eb79ed134f3a4e538e68119b35915015 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/eb79ed134f3a4e538e68119b35915015 2024-11-14T19:36:17,705 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/eb79ed134f3a4e538e68119b35915015, entries=150, sequenceid=655, filesize=12.0 K 2024-11-14T19:36:17,706 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/B/990efb78c1ed48dda153b21e495bcae3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/990efb78c1ed48dda153b21e495bcae3 2024-11-14T19:36:17,709 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/990efb78c1ed48dda153b21e495bcae3, entries=150, sequenceid=655, filesize=12.0 K 2024-11-14T19:36:17,710 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/.tmp/C/10769ff93be34d91bc28b171096968c8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/10769ff93be34d91bc28b171096968c8 2024-11-14T19:36:17,714 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/10769ff93be34d91bc28b171096968c8, entries=150, sequenceid=655, filesize=12.0 K 2024-11-14T19:36:17,715 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 47778d84645b4054cd2f50e381343edd in 864ms, sequenceid=655, compaction requested=true 2024-11-14T19:36:17,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1fef07a801ed4eae8bf28d3b86e867e4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/748c424ad65640d09baccce989ae250b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf2e31b6973f41a0abfa6fc09d838b13, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9d160d7ee9494203aa82cb915d1894f0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/5e261faa82b34dd5b67d6afd48def56c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/c9b40eda811543e69bdb894c654261e8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9f6aa9994776404b8811c0dfe07e903c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/07ef55d6f1f34e33a48b80268882fad8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf46fd9a17c04715979e950c383e69d2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/451e98dd6bd047a4967d67bfcda87238, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/58864f773e3e40ef942449041a26df70, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e4f061f9beb14131963ee617a66875ca, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/7b887b36b15145a4aa22a2c96cac08bc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/377e689ec0344446ab9c4f9320b93d4e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/05a7e69baf87429d88b77f992a7415ff, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e8eb2268aa1144a08f92b9174a025e06, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/fa72c751ebe94bfeae64b367a4d05845, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ba0a68e2e66a4b91b356c440f0616bae, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ebb13d5b32c24ffd9aec1bd43cc439a8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/06dab6d8cdcf42c88339807055494941, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/022dfd7032564b0aa706049327af1790] to archive 2024-11-14T19:36:17,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:17,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1fef07a801ed4eae8bf28d3b86e867e4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1fef07a801ed4eae8bf28d3b86e867e4 2024-11-14T19:36:17,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/748c424ad65640d09baccce989ae250b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/748c424ad65640d09baccce989ae250b 2024-11-14T19:36:17,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf2e31b6973f41a0abfa6fc09d838b13 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf2e31b6973f41a0abfa6fc09d838b13 2024-11-14T19:36:17,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9d160d7ee9494203aa82cb915d1894f0 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9d160d7ee9494203aa82cb915d1894f0 2024-11-14T19:36:17,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/5e261faa82b34dd5b67d6afd48def56c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/5e261faa82b34dd5b67d6afd48def56c 2024-11-14T19:36:17,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/c9b40eda811543e69bdb894c654261e8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/c9b40eda811543e69bdb894c654261e8 2024-11-14T19:36:17,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9f6aa9994776404b8811c0dfe07e903c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/9f6aa9994776404b8811c0dfe07e903c 2024-11-14T19:36:17,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/07ef55d6f1f34e33a48b80268882fad8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/07ef55d6f1f34e33a48b80268882fad8 2024-11-14T19:36:17,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf46fd9a17c04715979e950c383e69d2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/cf46fd9a17c04715979e950c383e69d2 2024-11-14T19:36:17,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/451e98dd6bd047a4967d67bfcda87238 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/451e98dd6bd047a4967d67bfcda87238 2024-11-14T19:36:17,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/58864f773e3e40ef942449041a26df70 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/58864f773e3e40ef942449041a26df70 2024-11-14T19:36:17,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e4f061f9beb14131963ee617a66875ca to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e4f061f9beb14131963ee617a66875ca 2024-11-14T19:36:17,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/7b887b36b15145a4aa22a2c96cac08bc to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/7b887b36b15145a4aa22a2c96cac08bc 2024-11-14T19:36:17,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/377e689ec0344446ab9c4f9320b93d4e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/377e689ec0344446ab9c4f9320b93d4e 2024-11-14T19:36:17,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/05a7e69baf87429d88b77f992a7415ff to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/05a7e69baf87429d88b77f992a7415ff 2024-11-14T19:36:17,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e8eb2268aa1144a08f92b9174a025e06 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/e8eb2268aa1144a08f92b9174a025e06 2024-11-14T19:36:17,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/fa72c751ebe94bfeae64b367a4d05845 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/fa72c751ebe94bfeae64b367a4d05845 2024-11-14T19:36:17,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ba0a68e2e66a4b91b356c440f0616bae to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ba0a68e2e66a4b91b356c440f0616bae 2024-11-14T19:36:17,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ebb13d5b32c24ffd9aec1bd43cc439a8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/ebb13d5b32c24ffd9aec1bd43cc439a8 2024-11-14T19:36:17,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/06dab6d8cdcf42c88339807055494941 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/06dab6d8cdcf42c88339807055494941 2024-11-14T19:36:17,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/022dfd7032564b0aa706049327af1790 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/022dfd7032564b0aa706049327af1790 2024-11-14T19:36:17,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/049e18e3951343789a51431b545fef1e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/22f43cbc77c2426292b8f8790a3cb9be, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/2fa20c780ae24a42bee80f5e46ecff6b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/f5529468bb5b4e63b01171115f5657fd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/62260b3b6b704705ad92b307a9f42e1f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/246d4ba0500e432fa41032fa28f19f91, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/61af9258b5d044d38d61749ec43951aa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3aeac0165a2c47a1a5198e8a75d411d4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/4055822feaf14deb80d77f321bbc2339, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/15850e53dea94e3a8e4f5dce08b739ff, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3f03156399e647e78dbcecdd8f2fc3d2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82a6cb34d01f4d65a807160b334e8e84, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/b8eb601113ce47eca63c507c42d9d052, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/867004c3ed33491aa1c2028ac26f830d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82807a9e681941199c166531f46e3481, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/1fd0b5ce93d2452fab2393c0c4ae4071, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/0fa5fb80d519441e90122f8fc14d6dc4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/a37e6a5e56574ad2bcf5b92e892bb1c7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/c43d701dfa474c3d9b6f43b2c27d1a63, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/7c7fe466ad4f4fe781c8de74b1b0dc8f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/26ebd0c9643a4b59935f8c8a6c8822fd] to archive 2024-11-14T19:36:17,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:17,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/049e18e3951343789a51431b545fef1e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/049e18e3951343789a51431b545fef1e 2024-11-14T19:36:17,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/22f43cbc77c2426292b8f8790a3cb9be to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/22f43cbc77c2426292b8f8790a3cb9be 2024-11-14T19:36:17,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/2fa20c780ae24a42bee80f5e46ecff6b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/2fa20c780ae24a42bee80f5e46ecff6b 2024-11-14T19:36:17,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/f5529468bb5b4e63b01171115f5657fd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/f5529468bb5b4e63b01171115f5657fd 2024-11-14T19:36:17,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/62260b3b6b704705ad92b307a9f42e1f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/62260b3b6b704705ad92b307a9f42e1f 2024-11-14T19:36:17,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/246d4ba0500e432fa41032fa28f19f91 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/246d4ba0500e432fa41032fa28f19f91 2024-11-14T19:36:17,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/61af9258b5d044d38d61749ec43951aa to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/61af9258b5d044d38d61749ec43951aa 2024-11-14T19:36:17,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3aeac0165a2c47a1a5198e8a75d411d4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3aeac0165a2c47a1a5198e8a75d411d4 2024-11-14T19:36:17,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/4055822feaf14deb80d77f321bbc2339 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/4055822feaf14deb80d77f321bbc2339 2024-11-14T19:36:17,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/15850e53dea94e3a8e4f5dce08b739ff to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/15850e53dea94e3a8e4f5dce08b739ff 2024-11-14T19:36:17,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3f03156399e647e78dbcecdd8f2fc3d2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/3f03156399e647e78dbcecdd8f2fc3d2 2024-11-14T19:36:17,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82a6cb34d01f4d65a807160b334e8e84 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82a6cb34d01f4d65a807160b334e8e84 2024-11-14T19:36:17,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/b8eb601113ce47eca63c507c42d9d052 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/b8eb601113ce47eca63c507c42d9d052 2024-11-14T19:36:17,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/867004c3ed33491aa1c2028ac26f830d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/867004c3ed33491aa1c2028ac26f830d 2024-11-14T19:36:17,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82807a9e681941199c166531f46e3481 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/82807a9e681941199c166531f46e3481 2024-11-14T19:36:17,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/1fd0b5ce93d2452fab2393c0c4ae4071 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/1fd0b5ce93d2452fab2393c0c4ae4071 2024-11-14T19:36:17,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/0fa5fb80d519441e90122f8fc14d6dc4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/0fa5fb80d519441e90122f8fc14d6dc4 2024-11-14T19:36:17,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/a37e6a5e56574ad2bcf5b92e892bb1c7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/a37e6a5e56574ad2bcf5b92e892bb1c7 2024-11-14T19:36:17,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/c43d701dfa474c3d9b6f43b2c27d1a63 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/c43d701dfa474c3d9b6f43b2c27d1a63 2024-11-14T19:36:17,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/7c7fe466ad4f4fe781c8de74b1b0dc8f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/7c7fe466ad4f4fe781c8de74b1b0dc8f 2024-11-14T19:36:17,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/26ebd0c9643a4b59935f8c8a6c8822fd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/26ebd0c9643a4b59935f8c8a6c8822fd 2024-11-14T19:36:17,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/437e8970fd164a1eaad5255a09f52843, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8e16ceddbf74ea5a9715442f2ae7dc8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f6bc3f92c0aa4a07ba89ae13d0123d4b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8c01d16d84c349849862a20d846b0ca5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/4d039298d96248fba61b900df355b93b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6d3647fdd358416987c2ac3cde6dd8e1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1cc8eb600aff47dc9bfd933ca26508f8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/ebf16398bb7e4b95a88dae0daf9b8504, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/54601378d8474ee08e4523c04f882635, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8d72ac7181b049cc8088f470abdb0546, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8e8f881933314fbfadb4f1e88efb736f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/879d14721b774e5dbd919fdeb9f2d68a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/cce0fb717f9d46d2b01bda48d3e273e1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1d1a9cc0d70745be98864bb383ea575d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/692384462f3642a2b70d91071687bbbf] to archive 2024-11-14T19:36:17,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:17,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/437e8970fd164a1eaad5255a09f52843 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/437e8970fd164a1eaad5255a09f52843 2024-11-14T19:36:17,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8e16ceddbf74ea5a9715442f2ae7dc8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8e16ceddbf74ea5a9715442f2ae7dc8 2024-11-14T19:36:17,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f6bc3f92c0aa4a07ba89ae13d0123d4b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f6bc3f92c0aa4a07ba89ae13d0123d4b 2024-11-14T19:36:17,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8c01d16d84c349849862a20d846b0ca5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8c01d16d84c349849862a20d846b0ca5 2024-11-14T19:36:17,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/4d039298d96248fba61b900df355b93b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/4d039298d96248fba61b900df355b93b 2024-11-14T19:36:17,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6d3647fdd358416987c2ac3cde6dd8e1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6d3647fdd358416987c2ac3cde6dd8e1 2024-11-14T19:36:17,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1cc8eb600aff47dc9bfd933ca26508f8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1cc8eb600aff47dc9bfd933ca26508f8 2024-11-14T19:36:17,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/ebf16398bb7e4b95a88dae0daf9b8504 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/ebf16398bb7e4b95a88dae0daf9b8504 2024-11-14T19:36:17,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/54601378d8474ee08e4523c04f882635 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/54601378d8474ee08e4523c04f882635 2024-11-14T19:36:17,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8d72ac7181b049cc8088f470abdb0546 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8d72ac7181b049cc8088f470abdb0546 2024-11-14T19:36:17,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8e8f881933314fbfadb4f1e88efb736f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/8e8f881933314fbfadb4f1e88efb736f 2024-11-14T19:36:17,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/879d14721b774e5dbd919fdeb9f2d68a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/879d14721b774e5dbd919fdeb9f2d68a 2024-11-14T19:36:17,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/cce0fb717f9d46d2b01bda48d3e273e1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/cce0fb717f9d46d2b01bda48d3e273e1 2024-11-14T19:36:17,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1d1a9cc0d70745be98864bb383ea575d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1d1a9cc0d70745be98864bb383ea575d 2024-11-14T19:36:17,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/692384462f3642a2b70d91071687bbbf to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/692384462f3642a2b70d91071687bbbf 2024-11-14T19:36:17,784 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/recovered.edits/658.seqid, newMaxSeqId=658, maxSeqId=1 2024-11-14T19:36:17,784 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. 2024-11-14T19:36:17,784 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for 47778d84645b4054cd2f50e381343edd: Waiting for close lock at 1731612976458Running coprocessor pre-close hooks at 1731612976458Disabling compacts and flushes for region at 1731612976458Disabling writes for close at 1731612976851 (+393 ms)Obtaining lock to block concurrent updates at 1731612976851Preparing flush snapshotting stores in 47778d84645b4054cd2f50e381343edd at 1731612976851Finished memstore snapshotting TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731612976852 (+1 ms)Flushing stores of TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd. at 1731612976852Flushing 47778d84645b4054cd2f50e381343edd/A: creating writer at 1731612976852Flushing 47778d84645b4054cd2f50e381343edd/A: appending metadata at 1731612976855 (+3 ms)Flushing 47778d84645b4054cd2f50e381343edd/A: closing flushed file at 1731612976855Flushing 47778d84645b4054cd2f50e381343edd/B: creating writer at 1731612977264 (+409 ms)Flushing 47778d84645b4054cd2f50e381343edd/B: appending metadata at 1731612977266 (+2 ms)Flushing 47778d84645b4054cd2f50e381343edd/B: closing flushed file at 1731612977266Flushing 47778d84645b4054cd2f50e381343edd/C: creating writer at 1731612977282 (+16 ms)Flushing 47778d84645b4054cd2f50e381343edd/C: appending metadata at 1731612977286 (+4 ms)Flushing 47778d84645b4054cd2f50e381343edd/C: closing flushed file at 1731612977286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4f2be2: reopening flushed file at 1731612977699 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52f9c073: reopening flushed file at 1731612977705 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c8eb9f4: reopening flushed file at 1731612977710 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 47778d84645b4054cd2f50e381343edd in 864ms, sequenceid=655, compaction requested=true at 1731612977715 (+5 ms)Writing region close event to WAL at 1731612977779 (+64 ms)Running coprocessor post-close hooks at 1731612977784 (+5 ms)Closed at 1731612977784 2024-11-14T19:36:17,786 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed 47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:17,787 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=47778d84645b4054cd2f50e381343edd, regionState=CLOSED 2024-11-14T19:36:17,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure 47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:36:17,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-14T19:36:17,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure 47778d84645b4054cd2f50e381343edd, server=867b237d0fa7,38601,1731612889637 in 1.4830 sec 2024-11-14T19:36:17,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-14T19:36:17,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=47778d84645b4054cd2f50e381343edd, UNASSIGN in 1.4890 sec 2024-11-14T19:36:17,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-14T19:36:17,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4910 sec 2024-11-14T19:36:17,793 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612977793"}]},"ts":"1731612977793"} 2024-11-14T19:36:17,794 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T19:36:17,794 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T19:36:17,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4990 sec 2024-11-14T19:36:18,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-14T19:36:18,443 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:18,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T19:36:18,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:18,445 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=84, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:18,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T19:36:18,446 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=84, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:18,448 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:18,450 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/recovered.edits] 2024-11-14T19:36:18,452 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1af015ec3d5f4d23babf9212f02ab4f7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/1af015ec3d5f4d23babf9212f02ab4f7 2024-11-14T19:36:18,453 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/eb79ed134f3a4e538e68119b35915015 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/A/eb79ed134f3a4e538e68119b35915015 2024-11-14T19:36:18,455 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/990efb78c1ed48dda153b21e495bcae3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/990efb78c1ed48dda153b21e495bcae3 2024-11-14T19:36:18,456 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/e51a849b69cc4d02b2ac4545ca31397a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/B/e51a849b69cc4d02b2ac4545ca31397a 2024-11-14T19:36:18,458 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/10769ff93be34d91bc28b171096968c8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/10769ff93be34d91bc28b171096968c8 2024-11-14T19:36:18,459 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1adc4d1b2eff4a00ac6f3bdc61da9452 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/1adc4d1b2eff4a00ac6f3bdc61da9452 2024-11-14T19:36:18,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6ed9cb62da52491d89cf30ccb79b0c01 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/6ed9cb62da52491d89cf30ccb79b0c01 2024-11-14T19:36:18,461 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/a94fe85deee44265add6e73639055ec1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/a94fe85deee44265add6e73639055ec1 2024-11-14T19:36:18,461 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8ad29a612cf47b484ef71ff2aa294ad to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/b8ad29a612cf47b484ef71ff2aa294ad 2024-11-14T19:36:18,462 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/d7c7a8604bda4ff4a6c6b41403d88b75 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/d7c7a8604bda4ff4a6c6b41403d88b75 2024-11-14T19:36:18,463 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f080426ede9546eea07f6d7b811ade4c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/C/f080426ede9546eea07f6d7b811ade4c 2024-11-14T19:36:18,467 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/recovered.edits/658.seqid to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd/recovered.edits/658.seqid 2024-11-14T19:36:18,467 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/47778d84645b4054cd2f50e381343edd 2024-11-14T19:36:18,468 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T19:36:18,470 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=84, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:18,473 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T19:36:18,475 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T19:36:18,476 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=84, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:18,476 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T19:36:18,476 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731612978476"}]},"ts":"9223372036854775807"} 2024-11-14T19:36:18,478 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T19:36:18,478 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 47778d84645b4054cd2f50e381343edd, NAME => 'TestAcidGuarantees,,1731612952935.47778d84645b4054cd2f50e381343edd.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T19:36:18,478 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T19:36:18,479 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731612978478"}]},"ts":"9223372036854775807"} 2024-11-14T19:36:18,480 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T19:36:18,482 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=84, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:18,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-14T19:36:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-14T19:36:18,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T19:36:18,553 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:18,564 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1421657582_22 at /127.0.0.1:57354 [Waiting for operation #145] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1742159255_22 at /127.0.0.1:43712 [Waiting for operation #276] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=576 (was 570) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5577 (was 5663) 2024-11-14T19:36:18,573 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=576, ProcessCount=11, AvailableMemoryMB=5577 2024-11-14T19:36:18,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:36:18,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:36:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:18,577 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T19:36:18,577 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:18,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 85 2024-11-14T19:36:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T19:36:18,578 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T19:36:18,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742128_1304 (size=1061) 2024-11-14T19:36:18,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T19:36:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T19:36:18,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T19:36:18,985 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:36:18,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742129_1305 (size=53) 2024-11-14T19:36:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T19:36:19,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:36:19,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d1f3fbc287ab8779d6c2fa31f746dbed, disabling compactions & flushes 2024-11-14T19:36:19,395 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. after waiting 0 ms 2024-11-14T19:36:19,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,395 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d1f3fbc287ab8779d6c2fa31f746dbed: Waiting for close lock at 1731612979395Disabling compacts and flushes for region at 1731612979395Disabling writes for close at 1731612979395Writing region close event to WAL at 1731612979395Closed at 1731612979395 2024-11-14T19:36:19,396 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T19:36:19,396 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731612979396"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731612979396"}]},"ts":"1731612979396"} 2024-11-14T19:36:19,398 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T19:36:19,399 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T19:36:19,399 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612979399"}]},"ts":"1731612979399"} 2024-11-14T19:36:19,401 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T19:36:19,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, ASSIGN}] 2024-11-14T19:36:19,403 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, ASSIGN 2024-11-14T19:36:19,404 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, ASSIGN; state=OFFLINE, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=false 2024-11-14T19:36:19,554 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:19,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, ASSIGN because future has completed 2024-11-14T19:36:19,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:36:19,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T19:36:19,791 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,791 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7752): Opening region: {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:36:19,792 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,792 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:36:19,792 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7794): checking encryption for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,792 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(7797): checking classloading for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,793 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,795 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:19,795 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f3fbc287ab8779d6c2fa31f746dbed columnFamilyName A 2024-11-14T19:36:19,795 DEBUG [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:19,795 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(327): Store=d1f3fbc287ab8779d6c2fa31f746dbed/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:19,795 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,796 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:19,797 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f3fbc287ab8779d6c2fa31f746dbed columnFamilyName B 2024-11-14T19:36:19,797 DEBUG [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:19,797 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(327): Store=d1f3fbc287ab8779d6c2fa31f746dbed/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:19,797 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,798 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:19,798 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f3fbc287ab8779d6c2fa31f746dbed columnFamilyName C 2024-11-14T19:36:19,798 DEBUG [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:19,799 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(327): Store=d1f3fbc287ab8779d6c2fa31f746dbed/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:19,799 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,799 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1038): replaying wal for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,800 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,801 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,802 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1048): stopping wal replay for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,802 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1060): Cleaning up temporary data for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,802 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:36:19,803 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1093): writing seq id for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,805 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:36:19,806 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1114): Opened d1f3fbc287ab8779d6c2fa31f746dbed; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62859280, jitterRate=-0.0633237361907959}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:36:19,806 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:19,806 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegion(1006): Region open journal for d1f3fbc287ab8779d6c2fa31f746dbed: Running coprocessor pre-open hook at 1731612979792Writing region info on filesystem at 1731612979792Initializing all the Stores at 1731612979793 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612979793Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612979793Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612979793Cleaning up temporary data from old regions at 1731612979802 (+9 ms)Running coprocessor post-open hooks at 1731612979806 (+4 ms)Region opened successfully at 1731612979806 2024-11-14T19:36:19,807 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., pid=87, masterSystemTime=1731612979787 2024-11-14T19:36:19,809 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,809 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=87}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:19,809 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=OPEN, openSeqNum=2, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:19,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:36:19,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-14T19:36:19,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 in 255 msec 2024-11-14T19:36:19,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-14T19:36:19,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, ASSIGN in 411 msec 2024-11-14T19:36:19,815 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T19:36:19,816 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731612979815"}]},"ts":"1731612979815"} 2024-11-14T19:36:19,817 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T19:36:19,818 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=85, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T19:36:19,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2430 sec 2024-11-14T19:36:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-14T19:36:20,724 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:20,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf9e372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:20,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:20,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:20,727 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f1f9c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:20,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:20,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6baf4ae5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:20,730 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:20,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:20,733 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T19:36:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 867b237d0fa7,38019,1731612888274 2024-11-14T19:36:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4dce7e46 2024-11-14T19:36:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T19:36:20,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T19:36:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:36:20,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:36:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:20,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742130_1306 (size=1097) 2024-11-14T19:36:21,160 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T19:36:21,160 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T19:36:21,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:36:21,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, REOPEN/MOVE}] 2024-11-14T19:36:21,165 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, REOPEN/MOVE 2024-11-14T19:36:21,166 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, REOPEN/MOVE because future has completed 2024-11-14T19:36:21,167 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:36:21,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:36:21,320 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,320 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:36:21,320 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing d1f3fbc287ab8779d6c2fa31f746dbed, disabling compactions & flushes 2024-11-14T19:36:21,320 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,320 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,320 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. after waiting 0 ms 2024-11-14T19:36:21,320 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,324 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T19:36:21,324 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,325 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for d1f3fbc287ab8779d6c2fa31f746dbed: Waiting for close lock at 1731612981320Running coprocessor pre-close hooks at 1731612981320Disabling compacts and flushes for region at 1731612981320Disabling writes for close at 1731612981320Writing region close event to WAL at 1731612981320Running coprocessor post-close hooks at 1731612981324 (+4 ms)Closed at 1731612981324 2024-11-14T19:36:21,325 WARN [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionServer(3299): Not adding moved region record: d1f3fbc287ab8779d6c2fa31f746dbed to self. 2024-11-14T19:36:21,327 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,327 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=CLOSED 2024-11-14T19:36:21,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:36:21,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-14T19:36:21,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 in 162 msec 2024-11-14T19:36:21,331 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, REOPEN/MOVE; state=CLOSED, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=true 2024-11-14T19:36:21,482 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, REOPEN/MOVE because future has completed 2024-11-14T19:36:21,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:36:21,638 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,638 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7752): Opening region: {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:36:21,639 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,639 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:36:21,639 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7794): checking encryption for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,639 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(7797): checking classloading for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,640 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,641 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:21,642 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f3fbc287ab8779d6c2fa31f746dbed columnFamilyName A 2024-11-14T19:36:21,643 DEBUG [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:21,644 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(327): Store=d1f3fbc287ab8779d6c2fa31f746dbed/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:21,644 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,645 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:21,645 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f3fbc287ab8779d6c2fa31f746dbed columnFamilyName B 2024-11-14T19:36:21,645 DEBUG [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:21,645 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(327): Store=d1f3fbc287ab8779d6c2fa31f746dbed/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:21,646 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,646 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:21,646 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1f3fbc287ab8779d6c2fa31f746dbed columnFamilyName C 2024-11-14T19:36:21,646 DEBUG [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:21,647 INFO [StoreOpener-d1f3fbc287ab8779d6c2fa31f746dbed-1 {}] regionserver.HStore(327): Store=d1f3fbc287ab8779d6c2fa31f746dbed/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:21,647 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,647 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1038): replaying wal for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,648 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,648 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,649 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1048): stopping wal replay for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,649 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1060): Cleaning up temporary data for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,649 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:36:21,651 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1093): writing seq id for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,652 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1114): Opened d1f3fbc287ab8779d6c2fa31f746dbed; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71996853, jitterRate=0.07283671200275421}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:36:21,652 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,653 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegion(1006): Region open journal for d1f3fbc287ab8779d6c2fa31f746dbed: Running coprocessor pre-open hook at 1731612981639Writing region info on filesystem at 1731612981639Initializing all the Stores at 1731612981640 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612981640Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612981640Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731612981640Cleaning up temporary data from old regions at 1731612981649 (+9 ms)Running coprocessor post-open hooks at 1731612981652 (+3 ms)Region opened successfully at 1731612981653 (+1 ms) 2024-11-14T19:36:21,653 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., pid=92, masterSystemTime=1731612981635 2024-11-14T19:36:21,655 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,655 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=92}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,656 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=OPEN, openSeqNum=5, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:36:21,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=90 2024-11-14T19:36:21,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 in 175 msec 2024-11-14T19:36:21,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-14T19:36:21,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, REOPEN/MOVE in 496 msec 2024-11-14T19:36:21,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-14T19:36:21,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-14T19:36:21,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-14T19:36:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-14T19:36:21,667 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:21,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e1ee1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,670 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d6c0fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c93267, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,674 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15622d18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,674 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,675 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21830194, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328c7170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25061205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,681 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523f95f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,683 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ada8f88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d28306f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614403d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,688 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112c3589, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,706 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29412344, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,706 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e0f4dfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@760e73bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c0c28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c883ef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8c861f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5109bfd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,721 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68269a6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,723 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bc5cad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@563dd2d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,725 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a202b11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c104ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c93281, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,732 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f3a211, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c6f1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5938121e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:21,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:21,744 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53c66cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:21,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:21,749 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:21,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25730b37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:21,751 DEBUG [Thread-1429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,755 DEBUG [Thread-1427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,756 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,758 DEBUG [Thread-1421 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,758 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,769 DEBUG [Thread-1412 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,773 DEBUG [Thread-1416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,774 DEBUG [Thread-1416 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,774 DEBUG [Thread-1416 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,775 DEBUG [Thread-1414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,784 DEBUG [Thread-1425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:21,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:36:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-14T19:36:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:21,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:21,794 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:21,795 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:21,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T19:36:21,800 DEBUG [Thread-1423 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,817 DEBUG [Thread-1410 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,845 DEBUG [Thread-1418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:21,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:21,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:21,848 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:21,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:36:21,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f90e150ad965428d83ba24eba60f4d60_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612981784/Put/seqid=0 2024-11-14T19:36:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731612981841, max timestamp=1731612981879Num uniques 859; 2024-11-14T19:36:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:36:21,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731612981841, max timestamp=1731612981879Num uniques 859; 2024-11-14T19:36:21,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:36:21,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731612981841, max timestamp=1731612981879Num uniques 859; 2024-11-14T19:36:21,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:36:21,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T19:36:21,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742131_1307 (size=14594) 2024-11-14T19:36:21,909 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:21,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114f90e150ad965428d83ba24eba60f4d60_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f90e150ad965428d83ba24eba60f4d60_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:21,915 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:21,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c is 175, key is test_row_0/A:col10/1731612981784/Put/seqid=0 2024-11-14T19:36:21,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T19:36:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:21,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:21,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:21,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613041925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613041939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:21,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742132_1308 (size=39549) 2024-11-14T19:36:21,959 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c 2024-11-14T19:36:21,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613041941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:21,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613041948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613041949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:21,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d18eed1e86474b6f82a7bfbdec9dcf66 is 50, key is test_row_0/B:col10/1731612981784/Put/seqid=0 2024-11-14T19:36:22,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613042053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613042053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613042063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742133_1309 (size=12001) 2024-11-14T19:36:22,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613042073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613042074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T19:36:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T19:36:22,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T19:36:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613042263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613042263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613042283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613042293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613042293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T19:36:22,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:22,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T19:36:22,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d18eed1e86474b6f82a7bfbdec9dcf66 2024-11-14T19:36:22,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/034c9f3ac5ec46979976dd2c04f051d2 is 50, key is test_row_0/C:col10/1731612981784/Put/seqid=0 2024-11-14T19:36:22,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742134_1310 (size=12001) 2024-11-14T19:36:22,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/034c9f3ac5ec46979976dd2c04f051d2 2024-11-14T19:36:22,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T19:36:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c 2024-11-14T19:36:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:22,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c, entries=200, sequenceid=15, filesize=38.6 K 2024-11-14T19:36:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613042583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613042583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d18eed1e86474b6f82a7bfbdec9dcf66 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d18eed1e86474b6f82a7bfbdec9dcf66 2024-11-14T19:36:22,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d18eed1e86474b6f82a7bfbdec9dcf66, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T19:36:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613042603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613042603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613042604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:22,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/034c9f3ac5ec46979976dd2c04f051d2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/034c9f3ac5ec46979976dd2c04f051d2 2024-11-14T19:36:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:22,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:22,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/034c9f3ac5ec46979976dd2c04f051d2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-14T19:36:22,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d1f3fbc287ab8779d6c2fa31f746dbed in 854ms, sequenceid=15, compaction requested=false 2024-11-14T19:36:22,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:22,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-14T19:36:22,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:22,725 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-14T19:36:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3b5d81b4e694249a784ff00c60b3a82_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612981939/Put/seqid=0 2024-11-14T19:36:22,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742135_1311 (size=12154) 2024-11-14T19:36:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T19:36:23,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:23,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613043123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613043130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613043132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613043134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613043136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:23,203 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c3b5d81b4e694249a784ff00c60b3a82_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3b5d81b4e694249a784ff00c60b3a82_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:23,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/4280638f6ed64fe18a92f50f9e84e5a5, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:23,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/4280638f6ed64fe18a92f50f9e84e5a5 is 175, key is test_row_0/A:col10/1731612981939/Put/seqid=0 2024-11-14T19:36:23,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742136_1312 (size=30955) 2024-11-14T19:36:23,251 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/4280638f6ed64fe18a92f50f9e84e5a5 2024-11-14T19:36:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613043243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613043253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/e355242f022f4c86b663bb66ef7f217b is 50, key is test_row_0/B:col10/1731612981939/Put/seqid=0 2024-11-14T19:36:23,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613043263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613043264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613043264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742137_1313 (size=12001) 2024-11-14T19:36:23,303 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/e355242f022f4c86b663bb66ef7f217b 2024-11-14T19:36:23,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f902fae6ce7848e8a0c44918c625b552 is 50, key is test_row_0/C:col10/1731612981939/Put/seqid=0 2024-11-14T19:36:23,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742138_1314 (size=12001) 2024-11-14T19:36:23,365 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f902fae6ce7848e8a0c44918c625b552 2024-11-14T19:36:23,370 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T19:36:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/4280638f6ed64fe18a92f50f9e84e5a5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/4280638f6ed64fe18a92f50f9e84e5a5 2024-11-14T19:36:23,390 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/4280638f6ed64fe18a92f50f9e84e5a5, entries=150, sequenceid=50, filesize=30.2 K 2024-11-14T19:36:23,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/e355242f022f4c86b663bb66ef7f217b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/e355242f022f4c86b663bb66ef7f217b 2024-11-14T19:36:23,396 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/e355242f022f4c86b663bb66ef7f217b, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T19:36:23,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f902fae6ce7848e8a0c44918c625b552 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f902fae6ce7848e8a0c44918c625b552 2024-11-14T19:36:23,407 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f902fae6ce7848e8a0c44918c625b552, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T19:36:23,409 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for d1f3fbc287ab8779d6c2fa31f746dbed in 684ms, sequenceid=50, compaction requested=false 2024-11-14T19:36:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:23,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-14T19:36:23,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-14T19:36:23,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-14T19:36:23,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-14T19:36:23,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.6270 sec 2024-11-14T19:36:23,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:36:23,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:23,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:23,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:23,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:23,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:23,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:23,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:23,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141b1fdb066c2d4866ac5bcd8d6bd4a460_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612983469/Put/seqid=0 2024-11-14T19:36:23,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742139_1315 (size=14594) 2024-11-14T19:36:23,541 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:23,545 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141b1fdb066c2d4866ac5bcd8d6bd4a460_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141b1fdb066c2d4866ac5bcd8d6bd4a460_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:23,547 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/46ead1675b4d4c8388ad08777e654f0c, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:23,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/46ead1675b4d4c8388ad08777e654f0c is 175, key is test_row_0/A:col10/1731612983469/Put/seqid=0 2024-11-14T19:36:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731612983522, max timestamp=1731612983553Num uniques 859; 2024-11-14T19:36:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T19:36:23,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16434, totalHeapSize=37576, min timestamp=1731612983522, max timestamp=1731612983553Num uniques 859; 2024-11-14T19:36:23,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613043545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,556 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=16434, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-14T19:36:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,557 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731612983524, max timestamp=1731612983553Num uniques 859; 2024-11-14T19:36:23,557 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:36:23,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742140_1316 (size=39549) 2024-11-14T19:36:23,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613043597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613043618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613043616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613043618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613043665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,679 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613043723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613043743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613043744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613043744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613043884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-14T19:36:23,943 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:23,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:23,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-14T19:36:23,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T19:36:23,947 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:23,948 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:23,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613043944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,960 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/46ead1675b4d4c8388ad08777e654f0c 2024-11-14T19:36:23,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613043964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613043964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:23,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613043964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:23,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:23,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/56dfa86db265496d85eb7a82667f60fb is 50, key is test_row_0/B:col10/1731612983469/Put/seqid=0 2024-11-14T19:36:24,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742141_1317 (size=12001) 2024-11-14T19:36:24,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T19:36:24,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T19:36:24,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613044213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T19:36:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T19:36:24,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613044273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613044293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613044293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613044293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T19:36:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/56dfa86db265496d85eb7a82667f60fb 2024-11-14T19:36:24,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/cb89ae19a12d4320966ff90399ef9c29 is 50, key is test_row_0/C:col10/1731612983469/Put/seqid=0 2024-11-14T19:36:24,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742142_1318 (size=12001) 2024-11-14T19:36:24,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T19:36:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T19:36:24,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613044733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T19:36:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:24,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613044794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,810 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613044813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613044813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:24,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613044824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:24,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/cb89ae19a12d4320966ff90399ef9c29 2024-11-14T19:36:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:24,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/46ead1675b4d4c8388ad08777e654f0c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/46ead1675b4d4c8388ad08777e654f0c 2024-11-14T19:36:24,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/46ead1675b4d4c8388ad08777e654f0c, entries=200, sequenceid=62, filesize=38.6 K 2024-11-14T19:36:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/56dfa86db265496d85eb7a82667f60fb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/56dfa86db265496d85eb7a82667f60fb 2024-11-14T19:36:24,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/56dfa86db265496d85eb7a82667f60fb, entries=150, sequenceid=62, filesize=11.7 K 2024-11-14T19:36:24,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/cb89ae19a12d4320966ff90399ef9c29 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cb89ae19a12d4320966ff90399ef9c29 2024-11-14T19:36:24,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cb89ae19a12d4320966ff90399ef9c29, entries=150, sequenceid=62, filesize=11.7 K 2024-11-14T19:36:24,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for d1f3fbc287ab8779d6c2fa31f746dbed in 1412ms, sequenceid=62, compaction requested=true 2024-11-14T19:36:24,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:24,884 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:24,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:24,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:24,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:24,884 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:24,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:24,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:24,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:24,885 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:24,885 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/A is initiating minor compaction (all files) 2024-11-14T19:36:24,885 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/A in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,885 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/4280638f6ed64fe18a92f50f9e84e5a5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/46ead1675b4d4c8388ad08777e654f0c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=107.5 K 2024-11-14T19:36:24,886 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,886 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/4280638f6ed64fe18a92f50f9e84e5a5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/46ead1675b4d4c8388ad08777e654f0c] 2024-11-14T19:36:24,887 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:24,887 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/B is initiating minor compaction (all files) 2024-11-14T19:36:24,887 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/B in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,887 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d18eed1e86474b6f82a7bfbdec9dcf66, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/e355242f022f4c86b663bb66ef7f217b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/56dfa86db265496d85eb7a82667f60fb] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.2 K 2024-11-14T19:36:24,888 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c2b17c5f4a41e9b1a21fd6eb9b7c7c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731612981784 2024-11-14T19:36:24,888 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting d18eed1e86474b6f82a7bfbdec9dcf66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731612981784 2024-11-14T19:36:24,888 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4280638f6ed64fe18a92f50f9e84e5a5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731612981939 2024-11-14T19:36:24,888 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting e355242f022f4c86b663bb66ef7f217b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731612981939 2024-11-14T19:36:24,889 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46ead1675b4d4c8388ad08777e654f0c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731612983121 2024-11-14T19:36:24,889 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 56dfa86db265496d85eb7a82667f60fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731612983121 2024-11-14T19:36:24,902 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:24,903 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/7e2ee6e99eab4b6099cec3933f503e2a is 50, key is test_row_0/B:col10/1731612983469/Put/seqid=0 2024-11-14T19:36:24,907 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:24,927 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149d2a4ff546694062b0be27f850945d70_d1f3fbc287ab8779d6c2fa31f746dbed store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:24,930 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149d2a4ff546694062b0be27f850945d70_d1f3fbc287ab8779d6c2fa31f746dbed, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:24,930 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149d2a4ff546694062b0be27f850945d70_d1f3fbc287ab8779d6c2fa31f746dbed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:24,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-14T19:36:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:24,946 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=127.47 KB heapSize=351 KB 2024-11-14T19:36:24,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:24,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742143_1319 (size=12104) 2024-11-14T19:36:24,991 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/7e2ee6e99eab4b6099cec3933f503e2a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/7e2ee6e99eab4b6099cec3933f503e2a 2024-11-14T19:36:25,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742144_1320 (size=4469) 2024-11-14T19:36:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144880cdd3f1bd44b7945053da522759fc_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612983615/Put/seqid=0 2024-11-14T19:36:25,010 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/B of d1f3fbc287ab8779d6c2fa31f746dbed into 7e2ee6e99eab4b6099cec3933f503e2a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:25,010 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:25,010 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/B, priority=13, startTime=1731612984884; duration=0sec 2024-11-14T19:36:25,010 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:25,010 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:B 2024-11-14T19:36:25,010 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:25,011 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:25,011 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/C is initiating minor compaction (all files) 2024-11-14T19:36:25,012 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/C in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:25,012 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/034c9f3ac5ec46979976dd2c04f051d2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f902fae6ce7848e8a0c44918c625b552, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cb89ae19a12d4320966ff90399ef9c29] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.2 K 2024-11-14T19:36:25,012 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 034c9f3ac5ec46979976dd2c04f051d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731612981784 2024-11-14T19:36:25,012 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f902fae6ce7848e8a0c44918c625b552, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731612981939 2024-11-14T19:36:25,012 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cb89ae19a12d4320966ff90399ef9c29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731612983121 2024-11-14T19:36:25,029 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:25,030 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f24f4454e91d413faf520096048c9d27 is 50, key is test_row_0/C:col10/1731612983469/Put/seqid=0 2024-11-14T19:36:25,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742145_1321 (size=12154) 2024-11-14T19:36:25,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:25,047 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144880cdd3f1bd44b7945053da522759fc_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144880cdd3f1bd44b7945053da522759fc_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:25,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/572c2ca3000843a1aafefc8ba008fe52, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/572c2ca3000843a1aafefc8ba008fe52 is 175, key is test_row_0/A:col10/1731612983615/Put/seqid=0 2024-11-14T19:36:25,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742146_1322 (size=12104) 2024-11-14T19:36:25,065 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f24f4454e91d413faf520096048c9d27 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f24f4454e91d413faf520096048c9d27 2024-11-14T19:36:25,072 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/C of d1f3fbc287ab8779d6c2fa31f746dbed into f24f4454e91d413faf520096048c9d27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:25,072 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:25,072 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/C, priority=13, startTime=1731612984884; duration=0sec 2024-11-14T19:36:25,072 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:25,072 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:C 2024-11-14T19:36:25,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T19:36:25,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742147_1323 (size=30955) 2024-11-14T19:36:25,406 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#A#compaction#267 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:25,406 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/94330c59b53c4db2bb71a00ddf569c28 is 175, key is test_row_0/A:col10/1731612983469/Put/seqid=0 2024-11-14T19:36:25,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742148_1324 (size=31058) 2024-11-14T19:36:25,502 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/572c2ca3000843a1aafefc8ba008fe52 2024-11-14T19:36:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/c762a943683c4ba9a591313d5f39d6e6 is 50, key is test_row_0/B:col10/1731612983615/Put/seqid=0 2024-11-14T19:36:25,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742149_1325 (size=12001) 2024-11-14T19:36:25,540 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/c762a943683c4ba9a591313d5f39d6e6 2024-11-14T19:36:25,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9 is 50, key is test_row_0/C:col10/1731612983615/Put/seqid=0 2024-11-14T19:36:25,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742150_1326 (size=12001) 2024-11-14T19:36:25,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:25,856 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/94330c59b53c4db2bb71a00ddf569c28 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/94330c59b53c4db2bb71a00ddf569c28 2024-11-14T19:36:25,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613045843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613045844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613045844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,860 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in d1f3fbc287ab8779d6c2fa31f746dbed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:36:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,860 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in d1f3fbc287ab8779d6c2fa31f746dbed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:36:25,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,865 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/A of d1f3fbc287ab8779d6c2fa31f746dbed into 94330c59b53c4db2bb71a00ddf569c28(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:25,865 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:25,865 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/A, priority=13, startTime=1731612984884; duration=0sec 2024-11-14T19:36:25,865 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:25,865 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:A 2024-11-14T19:36:25,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613045851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613045853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613045964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613045973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613045973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,990 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,991 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9 2024-11-14T19:36:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:25,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:25,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/572c2ca3000843a1aafefc8ba008fe52 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/572c2ca3000843a1aafefc8ba008fe52 2024-11-14T19:36:26,003 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/572c2ca3000843a1aafefc8ba008fe52, entries=150, sequenceid=95, filesize=30.2 K 2024-11-14T19:36:26,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/c762a943683c4ba9a591313d5f39d6e6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c762a943683c4ba9a591313d5f39d6e6 2024-11-14T19:36:26,011 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c762a943683c4ba9a591313d5f39d6e6, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T19:36:26,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9 2024-11-14T19:36:26,023 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9, entries=150, sequenceid=95, filesize=11.7 K 2024-11-14T19:36:26,025 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~350.95 KB/359376, currentSize=60.38 KB/61830 for d1f3fbc287ab8779d6c2fa31f746dbed in 1079ms, sequenceid=95, compaction requested=false 2024-11-14T19:36:26,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:26,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-14T19:36:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-14T19:36:26,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-14T19:36:26,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0900 sec 2024-11-14T19:36:26,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.0970 sec 2024-11-14T19:36:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-14T19:36:26,096 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:26,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-14T19:36:26,098 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T19:36:26,100 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:26,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:26,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:36:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T19:36:26,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143760a41f5d0447fe8ae07a58611794ef_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612985847/Put/seqid=0 2024-11-14T19:36:26,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742151_1327 (size=14594) 2024-11-14T19:36:26,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T19:36:26,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:26,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613046276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613046278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613046288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613046404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613046404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T19:36:26,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T19:36:26,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:26,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613046413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T19:36:26,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:26,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613046624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613046624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613046624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,633 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:26,639 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143760a41f5d0447fe8ae07a58611794ef_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143760a41f5d0447fe8ae07a58611794ef_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:26,640 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5da5cbb9da824aec9623e7a6d24f7440, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:26,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5da5cbb9da824aec9623e7a6d24f7440 is 175, key is test_row_0/A:col10/1731612985847/Put/seqid=0 2024-11-14T19:36:26,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742152_1328 (size=39549) 2024-11-14T19:36:26,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5da5cbb9da824aec9623e7a6d24f7440 2024-11-14T19:36:26,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d804f7611a1b4113bd10340749cc48cf is 50, key is test_row_0/B:col10/1731612985847/Put/seqid=0 2024-11-14T19:36:26,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742153_1329 (size=12001) 2024-11-14T19:36:26,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d804f7611a1b4113bd10340749cc48cf 2024-11-14T19:36:26,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/0a789d02fb144bcda12be8f34a03bee2 is 50, key is test_row_0/C:col10/1731612985847/Put/seqid=0 2024-11-14T19:36:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T19:36:26,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742154_1330 (size=12001) 2024-11-14T19:36:26,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T19:36:26,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:26,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/0a789d02fb144bcda12be8f34a03bee2 2024-11-14T19:36:26,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5da5cbb9da824aec9623e7a6d24f7440 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5da5cbb9da824aec9623e7a6d24f7440 2024-11-14T19:36:26,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5da5cbb9da824aec9623e7a6d24f7440, entries=200, sequenceid=111, filesize=38.6 K 2024-11-14T19:36:26,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d804f7611a1b4113bd10340749cc48cf as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d804f7611a1b4113bd10340749cc48cf 2024-11-14T19:36:26,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d804f7611a1b4113bd10340749cc48cf, entries=150, sequenceid=111, filesize=11.7 K 2024-11-14T19:36:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/0a789d02fb144bcda12be8f34a03bee2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/0a789d02fb144bcda12be8f34a03bee2 2024-11-14T19:36:26,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/0a789d02fb144bcda12be8f34a03bee2, entries=150, sequenceid=111, filesize=11.7 K 2024-11-14T19:36:26,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d1f3fbc287ab8779d6c2fa31f746dbed in 580ms, sequenceid=111, compaction requested=true 2024-11-14T19:36:26,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:26,771 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:26,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:26,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:26,771 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:26,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:26,772 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:26,772 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/B is initiating minor compaction (all files) 2024-11-14T19:36:26,772 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/B in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,772 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/7e2ee6e99eab4b6099cec3933f503e2a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c762a943683c4ba9a591313d5f39d6e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d804f7611a1b4113bd10340749cc48cf] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.3 K 2024-11-14T19:36:26,772 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:26,772 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/A is initiating minor compaction (all files) 2024-11-14T19:36:26,773 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/A in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,773 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/94330c59b53c4db2bb71a00ddf569c28, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/572c2ca3000843a1aafefc8ba008fe52, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5da5cbb9da824aec9623e7a6d24f7440] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=99.2 K 2024-11-14T19:36:26,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:26,773 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:26,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:26,773 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/94330c59b53c4db2bb71a00ddf569c28, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/572c2ca3000843a1aafefc8ba008fe52, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5da5cbb9da824aec9623e7a6d24f7440] 2024-11-14T19:36:26,773 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2ee6e99eab4b6099cec3933f503e2a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731612983121 2024-11-14T19:36:26,774 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94330c59b53c4db2bb71a00ddf569c28, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731612983121 2024-11-14T19:36:26,776 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c762a943683c4ba9a591313d5f39d6e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731612983612 2024-11-14T19:36:26,776 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 572c2ca3000843a1aafefc8ba008fe52, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731612983612 2024-11-14T19:36:26,777 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting d804f7611a1b4113bd10340749cc48cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731612985847 2024-11-14T19:36:26,777 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da5cbb9da824aec9623e7a6d24f7440, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731612985841 2024-11-14T19:36:26,800 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:26,802 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:26,802 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/3af65acc8aee498984ee469d85e49e13 is 50, key is test_row_0/B:col10/1731612985847/Put/seqid=0 2024-11-14T19:36:26,803 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114b8c910ba3dac4a4daa437a26c2acaeb3_d1f3fbc287ab8779d6c2fa31f746dbed store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:26,806 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114b8c910ba3dac4a4daa437a26c2acaeb3_d1f3fbc287ab8779d6c2fa31f746dbed, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:26,806 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b8c910ba3dac4a4daa437a26c2acaeb3_d1f3fbc287ab8779d6c2fa31f746dbed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742155_1331 (size=12207) 2024-11-14T19:36:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742156_1332 (size=4469) 2024-11-14T19:36:26,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-14T19:36:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:26,882 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T19:36:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:26,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:26,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465ca6c7443934a85a94cdf643624269a_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612986273/Put/seqid=0 2024-11-14T19:36:26,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742157_1333 (size=12204) 2024-11-14T19:36:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:26,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:26,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613046993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613046993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:26,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613046994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613047104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613047104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613047113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T19:36:27,236 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/3af65acc8aee498984ee469d85e49e13 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/3af65acc8aee498984ee469d85e49e13 2024-11-14T19:36:27,255 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/B of d1f3fbc287ab8779d6c2fa31f746dbed into 3af65acc8aee498984ee469d85e49e13(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:27,255 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:27,255 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/B, priority=13, startTime=1731612986771; duration=0sec 2024-11-14T19:36:27,255 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:27,255 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:B 2024-11-14T19:36:27,255 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:27,257 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#A#compaction#275 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:27,258 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e33713cade0249859d50823d51215ce5 is 175, key is test_row_0/A:col10/1731612985847/Put/seqid=0 2024-11-14T19:36:27,267 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:27,267 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/C is initiating minor compaction (all files) 2024-11-14T19:36:27,267 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/C in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:27,267 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f24f4454e91d413faf520096048c9d27, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/0a789d02fb144bcda12be8f34a03bee2] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.3 K 2024-11-14T19:36:27,271 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f24f4454e91d413faf520096048c9d27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731612983121 2024-11-14T19:36:27,272 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8f7ab5fafc48fcb33fec0f0f23d6c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731612983612 2024-11-14T19:36:27,273 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a789d02fb144bcda12be8f34a03bee2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731612985847 2024-11-14T19:36:27,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742158_1334 (size=31161) 2024-11-14T19:36:27,309 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e33713cade0249859d50823d51215ce5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e33713cade0249859d50823d51215ce5 2024-11-14T19:36:27,312 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:27,312 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/b5bfa314c31b4035a68949b0208e21a0 is 50, key is test_row_0/C:col10/1731612985847/Put/seqid=0 2024-11-14T19:36:27,321 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/A of d1f3fbc287ab8779d6c2fa31f746dbed into e33713cade0249859d50823d51215ce5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:27,321 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:27,321 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/A, priority=13, startTime=1731612986770; duration=0sec 2024-11-14T19:36:27,321 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:27,321 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:A 2024-11-14T19:36:27,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:27,335 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111465ca6c7443934a85a94cdf643624269a_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465ca6c7443934a85a94cdf643624269a_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/a6378904fc0742bc9cd0dc64cc4d8b79, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613047323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/a6378904fc0742bc9cd0dc64cc4d8b79 is 175, key is test_row_0/A:col10/1731612986273/Put/seqid=0 2024-11-14T19:36:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613047323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613047333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742159_1335 (size=12207) 2024-11-14T19:36:27,371 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/b5bfa314c31b4035a68949b0208e21a0 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b5bfa314c31b4035a68949b0208e21a0 2024-11-14T19:36:27,377 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/C of d1f3fbc287ab8779d6c2fa31f746dbed into b5bfa314c31b4035a68949b0208e21a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:27,377 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:27,377 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/C, priority=13, startTime=1731612986773; duration=0sec 2024-11-14T19:36:27,377 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:27,377 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:C 2024-11-14T19:36:27,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742160_1336 (size=31005) 2024-11-14T19:36:27,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613047643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613047643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613047653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,807 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/a6378904fc0742bc9cd0dc64cc4d8b79 2024-11-14T19:36:27,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/16928a442cad48538ad62cfc840be0d5 is 50, key is test_row_0/B:col10/1731612986273/Put/seqid=0 2024-11-14T19:36:27,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742161_1337 (size=12051) 2024-11-14T19:36:27,861 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/16928a442cad48538ad62cfc840be0d5 2024-11-14T19:36:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/7f2be06f67be40b1b7467ee025d8d448 is 50, key is test_row_0/C:col10/1731612986273/Put/seqid=0 2024-11-14T19:36:27,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613047875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,887 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in d1f3fbc287ab8779d6c2fa31f746dbed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:36:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613047883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:27,888 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in d1f3fbc287ab8779d6c2fa31f746dbed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:36:27,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:27,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:27,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742162_1338 (size=12051) 2024-11-14T19:36:27,950 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/7f2be06f67be40b1b7467ee025d8d448 2024-11-14T19:36:27,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/a6378904fc0742bc9cd0dc64cc4d8b79 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/a6378904fc0742bc9cd0dc64cc4d8b79 2024-11-14T19:36:27,961 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/a6378904fc0742bc9cd0dc64cc4d8b79, entries=150, sequenceid=130, filesize=30.3 K 2024-11-14T19:36:27,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/16928a442cad48538ad62cfc840be0d5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/16928a442cad48538ad62cfc840be0d5 2024-11-14T19:36:27,974 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/16928a442cad48538ad62cfc840be0d5, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T19:36:27,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/7f2be06f67be40b1b7467ee025d8d448 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f2be06f67be40b1b7467ee025d8d448 2024-11-14T19:36:27,984 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f2be06f67be40b1b7467ee025d8d448, entries=150, sequenceid=130, filesize=11.8 K 2024-11-14T19:36:27,985 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d1f3fbc287ab8779d6c2fa31f746dbed in 1103ms, sequenceid=130, compaction requested=false 2024-11-14T19:36:27,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:27,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:27,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-14T19:36:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-14T19:36:27,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-14T19:36:27,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-14T19:36:28,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8970 sec 2024-11-14T19:36:28,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T19:36:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:28,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114301560294e9a48c3b5ecc8227b2beb17_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612986991/Put/seqid=0 2024-11-14T19:36:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-14T19:36:28,244 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:28,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:28,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742163_1339 (size=14794) 2024-11-14T19:36:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-14T19:36:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T19:36:28,247 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:28,247 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:28,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:28,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613048270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613048270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613048273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T19:36:28,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:28,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:28,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613048404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613048404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613048404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T19:36:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613048624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613048624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613048624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,642 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:28,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114301560294e9a48c3b5ecc8227b2beb17_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114301560294e9a48c3b5ecc8227b2beb17_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:28,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b9bd72cdbb6a499f8ffa355cbd2696d3, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:28,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b9bd72cdbb6a499f8ffa355cbd2696d3 is 175, key is test_row_0/A:col10/1731612986991/Put/seqid=0 2024-11-14T19:36:28,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742164_1340 (size=39749) 2024-11-14T19:36:28,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:28,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:28,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:28,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T19:36:28,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T19:36:28,897 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-14T19:36:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613048943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613048943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613048943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:28,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:29,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,080 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b9bd72cdbb6a499f8ffa355cbd2696d3 2024-11-14T19:36:29,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/fd00712015554ffa9e46cd6676381020 is 50, key is test_row_0/B:col10/1731612986991/Put/seqid=0 2024-11-14T19:36:29,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742165_1341 (size=12151) 2024-11-14T19:36:29,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T19:36:29,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613049463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:29,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613049463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:29,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613049463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/fd00712015554ffa9e46cd6676381020 2024-11-14T19:36:29,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/65fe551f2f4348198902eb1392bb1f44 is 50, key is test_row_0/C:col10/1731612986991/Put/seqid=0 2024-11-14T19:36:29,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:29,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:29,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742166_1342 (size=12151) 2024-11-14T19:36:29,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/65fe551f2f4348198902eb1392bb1f44 2024-11-14T19:36:29,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b9bd72cdbb6a499f8ffa355cbd2696d3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b9bd72cdbb6a499f8ffa355cbd2696d3 2024-11-14T19:36:29,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b9bd72cdbb6a499f8ffa355cbd2696d3, entries=200, sequenceid=147, filesize=38.8 K 2024-11-14T19:36:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/fd00712015554ffa9e46cd6676381020 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/fd00712015554ffa9e46cd6676381020 2024-11-14T19:36:29,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/fd00712015554ffa9e46cd6676381020, entries=150, sequenceid=147, filesize=11.9 K 2024-11-14T19:36:29,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/65fe551f2f4348198902eb1392bb1f44 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/65fe551f2f4348198902eb1392bb1f44 2024-11-14T19:36:29,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/65fe551f2f4348198902eb1392bb1f44, entries=150, sequenceid=147, filesize=11.9 K 2024-11-14T19:36:29,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d1f3fbc287ab8779d6c2fa31f746dbed in 1529ms, sequenceid=147, compaction requested=true 2024-11-14T19:36:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:29,698 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:29,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:29,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T19:36:29,699 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:29,700 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:29,700 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/B is initiating minor compaction (all files) 2024-11-14T19:36:29,700 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/B in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,700 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/3af65acc8aee498984ee469d85e49e13, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/16928a442cad48538ad62cfc840be0d5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/fd00712015554ffa9e46cd6676381020] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.6 K 2024-11-14T19:36:29,701 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101915 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:29,701 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/A is initiating minor compaction (all files) 2024-11-14T19:36:29,701 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/A in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,701 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e33713cade0249859d50823d51215ce5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/a6378904fc0742bc9cd0dc64cc4d8b79, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b9bd72cdbb6a499f8ffa355cbd2696d3] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=99.5 K 2024-11-14T19:36:29,701 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,701 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e33713cade0249859d50823d51215ce5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/a6378904fc0742bc9cd0dc64cc4d8b79, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b9bd72cdbb6a499f8ffa355cbd2696d3] 2024-11-14T19:36:29,702 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af65acc8aee498984ee469d85e49e13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731612985847 2024-11-14T19:36:29,702 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e33713cade0249859d50823d51215ce5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731612985847 2024-11-14T19:36:29,702 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 16928a442cad48538ad62cfc840be0d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731612986273 2024-11-14T19:36:29,703 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6378904fc0742bc9cd0dc64cc4d8b79, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731612986273 2024-11-14T19:36:29,703 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting fd00712015554ffa9e46cd6676381020, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731612986991 2024-11-14T19:36:29,703 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9bd72cdbb6a499f8ffa355cbd2696d3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731612986991 2024-11-14T19:36:29,729 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#B#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:29,729 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/8c4b2ce67fad424d9574f918a630286c is 50, key is test_row_0/B:col10/1731612986991/Put/seqid=0 2024-11-14T19:36:29,737 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:29,756 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114c14f0924009142eeb8747e1b389e7312_d1f3fbc287ab8779d6c2fa31f746dbed store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:29,758 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114c14f0924009142eeb8747e1b389e7312_d1f3fbc287ab8779d6c2fa31f746dbed, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:29,758 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c14f0924009142eeb8747e1b389e7312_d1f3fbc287ab8779d6c2fa31f746dbed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:29,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-14T19:36:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:29,790 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:29,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742167_1343 (size=12459) 2024-11-14T19:36:29,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e83c0453dbed406e9defbd1c6d6728ae_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612988228/Put/seqid=0 2024-11-14T19:36:29,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742168_1344 (size=4469) 2024-11-14T19:36:29,849 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#A#compaction#285 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:29,850 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/1e5757fb185d4a98b71b8e5b52660f13 is 175, key is test_row_0/A:col10/1731612986991/Put/seqid=0 2024-11-14T19:36:29,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742169_1345 (size=12304) 2024-11-14T19:36:29,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742170_1346 (size=31413) 2024-11-14T19:36:30,206 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/8c4b2ce67fad424d9574f918a630286c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/8c4b2ce67fad424d9574f918a630286c 2024-11-14T19:36:30,212 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/B of d1f3fbc287ab8779d6c2fa31f746dbed into 8c4b2ce67fad424d9574f918a630286c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:30,212 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:30,212 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/B, priority=13, startTime=1731612989698; duration=0sec 2024-11-14T19:36:30,212 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:30,212 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:B 2024-11-14T19:36:30,212 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:30,214 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:30,215 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/C is initiating minor compaction (all files) 2024-11-14T19:36:30,215 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/C in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:30,215 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b5bfa314c31b4035a68949b0208e21a0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f2be06f67be40b1b7467ee025d8d448, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/65fe551f2f4348198902eb1392bb1f44] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.6 K 2024-11-14T19:36:30,216 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b5bfa314c31b4035a68949b0208e21a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731612985847 2024-11-14T19:36:30,216 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2be06f67be40b1b7467ee025d8d448, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731612986273 2024-11-14T19:36:30,216 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fe551f2f4348198902eb1392bb1f44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731612986991 2024-11-14T19:36:30,245 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#C#compaction#287 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:30,247 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/cea4175101454e67920ca0e8c8b73c03 is 50, key is test_row_0/C:col10/1731612986991/Put/seqid=0 2024-11-14T19:36:30,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742171_1347 (size=12459) 2024-11-14T19:36:30,298 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/cea4175101454e67920ca0e8c8b73c03 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cea4175101454e67920ca0e8c8b73c03 2024-11-14T19:36:30,310 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/C of d1f3fbc287ab8779d6c2fa31f746dbed into cea4175101454e67920ca0e8c8b73c03(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:30,310 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:30,310 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/C, priority=13, startTime=1731612989698; duration=0sec 2024-11-14T19:36:30,310 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:30,311 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:C 2024-11-14T19:36:30,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:30,338 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e83c0453dbed406e9defbd1c6d6728ae_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e83c0453dbed406e9defbd1c6d6728ae_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:30,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7fc1cade71484ea08be935f08caecd72, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:30,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7fc1cade71484ea08be935f08caecd72 is 175, key is test_row_0/A:col10/1731612988228/Put/seqid=0 2024-11-14T19:36:30,359 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/1e5757fb185d4a98b71b8e5b52660f13 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1e5757fb185d4a98b71b8e5b52660f13 2024-11-14T19:36:30,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742172_1348 (size=31105) 2024-11-14T19:36:30,373 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/A of d1f3fbc287ab8779d6c2fa31f746dbed into 1e5757fb185d4a98b71b8e5b52660f13(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:30,373 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:30,373 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/A, priority=13, startTime=1731612989698; duration=0sec 2024-11-14T19:36:30,374 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:30,374 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:A 2024-11-14T19:36:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T19:36:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:30,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613050516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613050522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613050525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613050633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613050644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613050644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,772 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7fc1cade71484ea08be935f08caecd72 2024-11-14T19:36:30,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b2f9f9bd801449d5a3511119abd9ee2c is 50, key is test_row_0/B:col10/1731612988228/Put/seqid=0 2024-11-14T19:36:30,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742173_1349 (size=12151) 2024-11-14T19:36:30,801 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b2f9f9bd801449d5a3511119abd9ee2c 2024-11-14T19:36:30,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/94d5131b8bd9492e8af61ca0a825a0c1 is 50, key is test_row_0/C:col10/1731612988228/Put/seqid=0 2024-11-14T19:36:30,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742174_1350 (size=12151) 2024-11-14T19:36:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613050843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,858 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/94d5131b8bd9492e8af61ca0a825a0c1 2024-11-14T19:36:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613050853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7fc1cade71484ea08be935f08caecd72 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7fc1cade71484ea08be935f08caecd72 2024-11-14T19:36:30,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613050863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:30,877 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7fc1cade71484ea08be935f08caecd72, entries=150, sequenceid=165, filesize=30.4 K 2024-11-14T19:36:30,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b2f9f9bd801449d5a3511119abd9ee2c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b2f9f9bd801449d5a3511119abd9ee2c 2024-11-14T19:36:30,887 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b2f9f9bd801449d5a3511119abd9ee2c, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T19:36:30,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/94d5131b8bd9492e8af61ca0a825a0c1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/94d5131b8bd9492e8af61ca0a825a0c1 2024-11-14T19:36:30,902 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/94d5131b8bd9492e8af61ca0a825a0c1, entries=150, sequenceid=165, filesize=11.9 K 2024-11-14T19:36:30,903 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d1f3fbc287ab8779d6c2fa31f746dbed in 1114ms, sequenceid=165, compaction requested=false 2024-11-14T19:36:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:30,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-14T19:36:30,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-14T19:36:30,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-14T19:36:30,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6580 sec 2024-11-14T19:36:30,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.6720 sec 2024-11-14T19:36:31,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:36:31,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:31,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:31,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:31,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:31,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:31,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:31,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142476aeddcc9a42de93b95f7e8722b88f_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612991173/Put/seqid=0 2024-11-14T19:36:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742175_1351 (size=14794) 2024-11-14T19:36:31,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:31,248 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142476aeddcc9a42de93b95f7e8722b88f_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142476aeddcc9a42de93b95f7e8722b88f_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:31,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7424eed8634144d2912ebaa9e89e3eb5, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:31,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7424eed8634144d2912ebaa9e89e3eb5 is 175, key is test_row_0/A:col10/1731612991173/Put/seqid=0 2024-11-14T19:36:31,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613051257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742176_1352 (size=39749) 2024-11-14T19:36:31,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613051259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613051260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613051383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613051383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613051383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613051603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613051603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613051603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,678 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=183, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7424eed8634144d2912ebaa9e89e3eb5 2024-11-14T19:36:31,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b722a86c58af431b824ed251e8f907a7 is 50, key is test_row_0/B:col10/1731612991173/Put/seqid=0 2024-11-14T19:36:31,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742177_1353 (size=12151) 2024-11-14T19:36:31,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b722a86c58af431b824ed251e8f907a7 2024-11-14T19:36:31,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/528e2d50baff446794aa91fda77e2ff8 is 50, key is test_row_0/C:col10/1731612991173/Put/seqid=0 2024-11-14T19:36:31,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742178_1354 (size=12151) 2024-11-14T19:36:31,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/528e2d50baff446794aa91fda77e2ff8 2024-11-14T19:36:31,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/7424eed8634144d2912ebaa9e89e3eb5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7424eed8634144d2912ebaa9e89e3eb5 2024-11-14T19:36:31,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7424eed8634144d2912ebaa9e89e3eb5, entries=200, sequenceid=183, filesize=38.8 K 2024-11-14T19:36:31,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b722a86c58af431b824ed251e8f907a7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b722a86c58af431b824ed251e8f907a7 2024-11-14T19:36:31,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b722a86c58af431b824ed251e8f907a7, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T19:36:31,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/528e2d50baff446794aa91fda77e2ff8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/528e2d50baff446794aa91fda77e2ff8 2024-11-14T19:36:31,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/528e2d50baff446794aa91fda77e2ff8, entries=150, sequenceid=183, filesize=11.9 K 2024-11-14T19:36:31,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d1f3fbc287ab8779d6c2fa31f746dbed in 633ms, sequenceid=183, compaction requested=true 2024-11-14T19:36:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:31,808 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:31,808 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:31,812 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:31,812 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/B is initiating minor compaction (all files) 2024-11-14T19:36:31,812 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/B in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:31,812 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/8c4b2ce67fad424d9574f918a630286c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b2f9f9bd801449d5a3511119abd9ee2c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b722a86c58af431b824ed251e8f907a7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.9 K 2024-11-14T19:36:31,813 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:31,813 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/A is initiating minor compaction (all files) 2024-11-14T19:36:31,813 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/A in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:31,813 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c4b2ce67fad424d9574f918a630286c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731612986991 2024-11-14T19:36:31,813 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1e5757fb185d4a98b71b8e5b52660f13, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7fc1cade71484ea08be935f08caecd72, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7424eed8634144d2912ebaa9e89e3eb5] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=99.9 K 2024-11-14T19:36:31,813 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:31,813 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1e5757fb185d4a98b71b8e5b52660f13, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7fc1cade71484ea08be935f08caecd72, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7424eed8634144d2912ebaa9e89e3eb5] 2024-11-14T19:36:31,814 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e5757fb185d4a98b71b8e5b52660f13, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731612986991 2024-11-14T19:36:31,814 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f9f9bd801449d5a3511119abd9ee2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731612988228 2024-11-14T19:36:31,815 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc1cade71484ea08be935f08caecd72, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731612988228 2024-11-14T19:36:31,815 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b722a86c58af431b824ed251e8f907a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731612990507 2024-11-14T19:36:31,816 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7424eed8634144d2912ebaa9e89e3eb5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731612990507 2024-11-14T19:36:31,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:31,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:31,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:31,831 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:31,839 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:31,840 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/92c15476a44a448399533d3aa495ce45 is 50, key is test_row_0/B:col10/1731612991173/Put/seqid=0 2024-11-14T19:36:31,841 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114448dfde815f24e0eb696ac85065fb685_d1f3fbc287ab8779d6c2fa31f746dbed store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:31,843 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114448dfde815f24e0eb696ac85065fb685_d1f3fbc287ab8779d6c2fa31f746dbed, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:31,843 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114448dfde815f24e0eb696ac85065fb685_d1f3fbc287ab8779d6c2fa31f746dbed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:31,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742180_1356 (size=4469) 2024-11-14T19:36:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:31,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:31,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742179_1355 (size=12561) 2024-11-14T19:36:31,927 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#A#compaction#293 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:31,927 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/8ad01ecf10ba47a5a638391ad63d6473 is 175, key is test_row_0/A:col10/1731612991173/Put/seqid=0 2024-11-14T19:36:31,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613051945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613051948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613051956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613051957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742181_1357 (size=31515) 2024-11-14T19:36:31,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a3c1a105097e45debd6c42b8581ade06_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612991920/Put/seqid=0 2024-11-14T19:36:31,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613051962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:31,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:31,989 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/8ad01ecf10ba47a5a638391ad63d6473 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8ad01ecf10ba47a5a638391ad63d6473 2024-11-14T19:36:32,002 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/A of d1f3fbc287ab8779d6c2fa31f746dbed into 8ad01ecf10ba47a5a638391ad63d6473(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:32,002 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:32,002 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/A, priority=13, startTime=1731612991808; duration=0sec 2024-11-14T19:36:32,002 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:32,002 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:A 2024-11-14T19:36:32,002 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:32,003 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:32,004 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/C is initiating minor compaction (all files) 2024-11-14T19:36:32,004 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/C in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:32,004 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cea4175101454e67920ca0e8c8b73c03, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/94d5131b8bd9492e8af61ca0a825a0c1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/528e2d50baff446794aa91fda77e2ff8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=35.9 K 2024-11-14T19:36:32,004 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea4175101454e67920ca0e8c8b73c03, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731612986991 2024-11-14T19:36:32,004 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94d5131b8bd9492e8af61ca0a825a0c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731612988228 2024-11-14T19:36:32,005 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 528e2d50baff446794aa91fda77e2ff8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731612990507 2024-11-14T19:36:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742182_1358 (size=14794) 2024-11-14T19:36:32,016 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#C#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:32,016 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/e0a6d9de59b24d58ae02b6ba49af1eda is 50, key is test_row_0/C:col10/1731612991173/Put/seqid=0 2024-11-14T19:36:32,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742183_1359 (size=12561) 2024-11-14T19:36:32,053 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/e0a6d9de59b24d58ae02b6ba49af1eda as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/e0a6d9de59b24d58ae02b6ba49af1eda 2024-11-14T19:36:32,064 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/C of d1f3fbc287ab8779d6c2fa31f746dbed into e0a6d9de59b24d58ae02b6ba49af1eda(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:32,064 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:32,064 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/C, priority=13, startTime=1731612991822; duration=0sec 2024-11-14T19:36:32,065 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:32,065 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:C 2024-11-14T19:36:32,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613052063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613052063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613052073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613052073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613052084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613052284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613052284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613052284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613052284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613052303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,328 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/92c15476a44a448399533d3aa495ce45 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/92c15476a44a448399533d3aa495ce45 2024-11-14T19:36:32,335 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/B of d1f3fbc287ab8779d6c2fa31f746dbed into 92c15476a44a448399533d3aa495ce45(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:32,335 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:32,335 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/B, priority=13, startTime=1731612991808; duration=0sec 2024-11-14T19:36:32,335 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:32,335 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:B 2024-11-14T19:36:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-14T19:36:32,414 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:32,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:32,415 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-14T19:36:32,416 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T19:36:32,417 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:32,417 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:32,422 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a3c1a105097e45debd6c42b8581ade06_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3c1a105097e45debd6c42b8581ade06_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:32,423 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/06f47d47caea44cbbfbaa51f533178f8, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:32,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/06f47d47caea44cbbfbaa51f533178f8 is 175, key is test_row_0/A:col10/1731612991920/Put/seqid=0 2024-11-14T19:36:32,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742184_1360 (size=39749) 2024-11-14T19:36:32,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/06f47d47caea44cbbfbaa51f533178f8 2024-11-14T19:36:32,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa is 50, key is test_row_0/B:col10/1731612991920/Put/seqid=0 2024-11-14T19:36:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T19:36:32,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742185_1361 (size=12151) 2024-11-14T19:36:32,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T19:36:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:32,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613052604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613052604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613052604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613052604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613052615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:32,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:32,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T19:36:32,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:32,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:32,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T19:36:32,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T19:36:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:32,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:32,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa 2024-11-14T19:36:32,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/958a530de1a4490498930d0c79c893a6 is 50, key is test_row_0/C:col10/1731612991920/Put/seqid=0 2024-11-14T19:36:32,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742186_1362 (size=12151) 2024-11-14T19:36:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T19:36:33,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T19:36:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:33,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613053123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613053123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613053123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613053123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613053123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:33,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:33,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T19:36:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:33,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T19:36:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:33,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:33,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/958a530de1a4490498930d0c79c893a6 2024-11-14T19:36:33,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/06f47d47caea44cbbfbaa51f533178f8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/06f47d47caea44cbbfbaa51f533178f8 2024-11-14T19:36:33,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/06f47d47caea44cbbfbaa51f533178f8, entries=200, sequenceid=201, filesize=38.8 K 2024-11-14T19:36:33,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa 2024-11-14T19:36:33,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa, entries=150, sequenceid=201, filesize=11.9 K 2024-11-14T19:36:33,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/958a530de1a4490498930d0c79c893a6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/958a530de1a4490498930d0c79c893a6 2024-11-14T19:36:33,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/958a530de1a4490498930d0c79c893a6, entries=150, sequenceid=201, filesize=11.9 K 2024-11-14T19:36:33,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d1f3fbc287ab8779d6c2fa31f746dbed in 1476ms, sequenceid=201, compaction requested=false 2024-11-14T19:36:33,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:33,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-14T19:36:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:33,504 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-14T19:36:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:33,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146182c0f9d1ef43d79f92bf0352f1f0cc_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612991935/Put/seqid=0 2024-11-14T19:36:33,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742187_1363 (size=12304) 2024-11-14T19:36:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:33,527 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411146182c0f9d1ef43d79f92bf0352f1f0cc_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146182c0f9d1ef43d79f92bf0352f1f0cc_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:33,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/10e5376bb3364384a199b65722841bd1, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:33,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/10e5376bb3364384a199b65722841bd1 is 175, key is test_row_0/A:col10/1731612991935/Put/seqid=0 2024-11-14T19:36:33,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742188_1364 (size=31105) 2024-11-14T19:36:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T19:36:33,937 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/10e5376bb3364384a199b65722841bd1 2024-11-14T19:36:33,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/4bde6a14fb034a179756412a2d7fc920 is 50, key is test_row_0/B:col10/1731612991935/Put/seqid=0 2024-11-14T19:36:33,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742189_1365 (size=12151) 2024-11-14T19:36:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:34,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:34,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613054153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613054154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613054156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613054162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613054163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613054273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613054273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613054273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613054273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613054283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,349 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/4bde6a14fb034a179756412a2d7fc920 2024-11-14T19:36:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/4217b49e520f498c8ad9cd6368cb91b3 is 50, key is test_row_0/C:col10/1731612991935/Put/seqid=0 2024-11-14T19:36:34,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742190_1366 (size=12151) 2024-11-14T19:36:34,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613054493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613054493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613054494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613054494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613054494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T19:36:34,760 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/4217b49e520f498c8ad9cd6368cb91b3 2024-11-14T19:36:34,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/10e5376bb3364384a199b65722841bd1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/10e5376bb3364384a199b65722841bd1 2024-11-14T19:36:34,767 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/10e5376bb3364384a199b65722841bd1, entries=150, sequenceid=218, filesize=30.4 K 2024-11-14T19:36:34,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/4bde6a14fb034a179756412a2d7fc920 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/4bde6a14fb034a179756412a2d7fc920 2024-11-14T19:36:34,772 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/4bde6a14fb034a179756412a2d7fc920, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T19:36:34,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/4217b49e520f498c8ad9cd6368cb91b3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/4217b49e520f498c8ad9cd6368cb91b3 2024-11-14T19:36:34,777 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/4217b49e520f498c8ad9cd6368cb91b3, entries=150, sequenceid=218, filesize=11.9 K 2024-11-14T19:36:34,778 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for d1f3fbc287ab8779d6c2fa31f746dbed in 1273ms, sequenceid=218, compaction requested=true 2024-11-14T19:36:34,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:34,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:34,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-14T19:36:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-14T19:36:34,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-14T19:36:34,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3620 sec 2024-11-14T19:36:34,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3680 sec 2024-11-14T19:36:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:34,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-14T19:36:34,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:34,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:34,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:34,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114095f49a32f2741d6ac358a070d82b413_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612994153/Put/seqid=0 2024-11-14T19:36:34,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742191_1367 (size=17284) 2024-11-14T19:36:34,835 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:34,838 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114095f49a32f2741d6ac358a070d82b413_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114095f49a32f2741d6ac358a070d82b413_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613054827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e522f3620279409b86b3d1c1184f65dd, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613054834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e522f3620279409b86b3d1c1184f65dd is 175, key is test_row_0/A:col10/1731612994153/Put/seqid=0 2024-11-14T19:36:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742192_1368 (size=48389) 2024-11-14T19:36:34,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613054835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613054836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613054838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613054943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613054943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613054963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613054963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613054963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613055153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613055153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613055183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613055183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613055184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,243 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e522f3620279409b86b3d1c1184f65dd 2024-11-14T19:36:35,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/21dad3ca437a489cb0c692610cfa3f27 is 50, key is test_row_0/B:col10/1731612994153/Put/seqid=0 2024-11-14T19:36:35,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742193_1369 (size=12151) 2024-11-14T19:36:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613055473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613055473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613055493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613055493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613055503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/21dad3ca437a489cb0c692610cfa3f27 2024-11-14T19:36:35,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/576d9d63c1604503b62bc2d86a0e9d16 is 50, key is test_row_0/C:col10/1731612994153/Put/seqid=0 2024-11-14T19:36:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742194_1370 (size=12151) 2024-11-14T19:36:35,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613055993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613055993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:36,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613056003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:36,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:36,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613056013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:36,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/576d9d63c1604503b62bc2d86a0e9d16 2024-11-14T19:36:36,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e522f3620279409b86b3d1c1184f65dd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e522f3620279409b86b3d1c1184f65dd 2024-11-14T19:36:36,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e522f3620279409b86b3d1c1184f65dd, entries=250, sequenceid=238, filesize=47.3 K 2024-11-14T19:36:36,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/21dad3ca437a489cb0c692610cfa3f27 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/21dad3ca437a489cb0c692610cfa3f27 2024-11-14T19:36:36,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/21dad3ca437a489cb0c692610cfa3f27, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T19:36:36,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/576d9d63c1604503b62bc2d86a0e9d16 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/576d9d63c1604503b62bc2d86a0e9d16 2024-11-14T19:36:36,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/576d9d63c1604503b62bc2d86a0e9d16, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T19:36:36,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d1f3fbc287ab8779d6c2fa31f746dbed in 1264ms, sequenceid=238, compaction requested=true 2024-11-14T19:36:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:36,082 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:36:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:36,082 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:36:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:36,083 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150758 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:36:36,083 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:36:36,083 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/A is initiating minor compaction (all files) 2024-11-14T19:36:36,083 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/B is initiating minor compaction (all files) 2024-11-14T19:36:36,083 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/B in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:36,083 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/A in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:36,084 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8ad01ecf10ba47a5a638391ad63d6473, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/06f47d47caea44cbbfbaa51f533178f8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/10e5376bb3364384a199b65722841bd1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e522f3620279409b86b3d1c1184f65dd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=147.2 K 2024-11-14T19:36:36,084 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/92c15476a44a448399533d3aa495ce45, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/4bde6a14fb034a179756412a2d7fc920, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/21dad3ca437a489cb0c692610cfa3f27] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=47.9 K 2024-11-14T19:36:36,084 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:36,084 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8ad01ecf10ba47a5a638391ad63d6473, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/06f47d47caea44cbbfbaa51f533178f8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/10e5376bb3364384a199b65722841bd1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e522f3620279409b86b3d1c1184f65dd] 2024-11-14T19:36:36,084 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c15476a44a448399533d3aa495ce45, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731612990507 2024-11-14T19:36:36,084 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ad01ecf10ba47a5a638391ad63d6473, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731612990507 2024-11-14T19:36:36,084 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a7ed5f5e4c64a3aa5e10c2509d0cffa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731612991252 2024-11-14T19:36:36,084 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f47d47caea44cbbfbaa51f533178f8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731612991252 2024-11-14T19:36:36,084 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bde6a14fb034a179756412a2d7fc920, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731612991935 2024-11-14T19:36:36,085 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e5376bb3364384a199b65722841bd1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731612991935 2024-11-14T19:36:36,085 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 21dad3ca437a489cb0c692610cfa3f27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731612994153 2024-11-14T19:36:36,085 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e522f3620279409b86b3d1c1184f65dd, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731612994153 2024-11-14T19:36:36,097 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:36,097 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#B#compaction#305 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:36,097 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/1d29076ebae447db9e49247211dd3506 is 50, key is test_row_0/B:col10/1731612994153/Put/seqid=0 2024-11-14T19:36:36,101 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114451a1f35b5f9422abdec9ddcbc51932a_d1f3fbc287ab8779d6c2fa31f746dbed store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:36,104 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114451a1f35b5f9422abdec9ddcbc51932a_d1f3fbc287ab8779d6c2fa31f746dbed, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:36,104 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114451a1f35b5f9422abdec9ddcbc51932a_d1f3fbc287ab8779d6c2fa31f746dbed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:36,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742195_1371 (size=12697) 2024-11-14T19:36:36,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742196_1372 (size=4469) 2024-11-14T19:36:36,510 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/1d29076ebae447db9e49247211dd3506 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1d29076ebae447db9e49247211dd3506 2024-11-14T19:36:36,514 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/B of d1f3fbc287ab8779d6c2fa31f746dbed into 1d29076ebae447db9e49247211dd3506(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:36,514 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:36,514 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/B, priority=12, startTime=1731612996082; duration=0sec 2024-11-14T19:36:36,515 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:36,515 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:B 2024-11-14T19:36:36,515 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:36:36,515 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#A#compaction#306 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:36,515 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/3b6ab989fa184353800c171254d0b81f is 175, key is test_row_0/A:col10/1731612994153/Put/seqid=0 2024-11-14T19:36:36,516 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:36:36,516 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/C is initiating minor compaction (all files) 2024-11-14T19:36:36,516 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/C in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:36,516 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/e0a6d9de59b24d58ae02b6ba49af1eda, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/958a530de1a4490498930d0c79c893a6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/4217b49e520f498c8ad9cd6368cb91b3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/576d9d63c1604503b62bc2d86a0e9d16] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=47.9 K 2024-11-14T19:36:36,517 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a6d9de59b24d58ae02b6ba49af1eda, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731612990507 2024-11-14T19:36:36,517 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 958a530de1a4490498930d0c79c893a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731612991252 2024-11-14T19:36:36,517 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4217b49e520f498c8ad9cd6368cb91b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731612991935 2024-11-14T19:36:36,518 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 576d9d63c1604503b62bc2d86a0e9d16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731612994153 2024-11-14T19:36:36,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742197_1373 (size=31651) 2024-11-14T19:36:36,524 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:36,525 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/7f3c6f8e4d38499b9b3343ab9c4138d3 is 50, key is test_row_0/C:col10/1731612994153/Put/seqid=0 2024-11-14T19:36:36,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742198_1374 (size=12697) 2024-11-14T19:36:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-14T19:36:36,593 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:36,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-14T19:36:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T19:36:36,595 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:36,596 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:36,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T19:36:36,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-14T19:36:36,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:36,748 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:36:36,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:36,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb0fc3f1f0be4094b57ec0b78a60d8cf_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612994832/Put/seqid=0 2024-11-14T19:36:36,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742199_1375 (size=12304) 2024-11-14T19:36:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T19:36:36,924 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/3b6ab989fa184353800c171254d0b81f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/3b6ab989fa184353800c171254d0b81f 2024-11-14T19:36:36,928 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/A of d1f3fbc287ab8779d6c2fa31f746dbed into 3b6ab989fa184353800c171254d0b81f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:36,928 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:36,928 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/A, priority=12, startTime=1731612996082; duration=0sec 2024-11-14T19:36:36,928 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:36,928 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:A 2024-11-14T19:36:36,933 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/7f3c6f8e4d38499b9b3343ab9c4138d3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f3c6f8e4d38499b9b3343ab9c4138d3 2024-11-14T19:36:36,937 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/C of d1f3fbc287ab8779d6c2fa31f746dbed into 7f3c6f8e4d38499b9b3343ab9c4138d3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:36,937 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:36,937 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/C, priority=12, startTime=1731612996082; duration=0sec 2024-11-14T19:36:36,937 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:36,937 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:C 2024-11-14T19:36:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:37,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:37,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613057042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613057043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613057044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613057045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613057046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:37,163 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb0fc3f1f0be4094b57ec0b78a60d8cf_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb0fc3f1f0be4094b57ec0b78a60d8cf_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/1307cad13fc248d0b7e358c087ab6ae5, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/1307cad13fc248d0b7e358c087ab6ae5 is 175, key is test_row_0/A:col10/1731612994832/Put/seqid=0 2024-11-14T19:36:37,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742200_1376 (size=31105) 2024-11-14T19:36:37,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613057163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613057163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613057163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613057163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613057163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T19:36:37,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613057383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613057383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613057383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613057383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613057383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,568 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/1307cad13fc248d0b7e358c087ab6ae5 2024-11-14T19:36:37,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/1450b3f0250f43dcabc15c35c586fa07 is 50, key is test_row_0/B:col10/1731612994832/Put/seqid=0 2024-11-14T19:36:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742201_1377 (size=12151) 2024-11-14T19:36:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613057693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613057693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613057693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613057693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613057693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:37,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T19:36:37,979 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/1450b3f0250f43dcabc15c35c586fa07 2024-11-14T19:36:37,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/3e0d858a59674344819abd941f1b66ec is 50, key is test_row_0/C:col10/1731612994832/Put/seqid=0 2024-11-14T19:36:37,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742202_1378 (size=12151) 2024-11-14T19:36:37,992 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/3e0d858a59674344819abd941f1b66ec 2024-11-14T19:36:37,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/1307cad13fc248d0b7e358c087ab6ae5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1307cad13fc248d0b7e358c087ab6ae5 2024-11-14T19:36:38,000 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1307cad13fc248d0b7e358c087ab6ae5, entries=150, sequenceid=251, filesize=30.4 K 2024-11-14T19:36:38,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/1450b3f0250f43dcabc15c35c586fa07 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1450b3f0250f43dcabc15c35c586fa07 2024-11-14T19:36:38,003 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1450b3f0250f43dcabc15c35c586fa07, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T19:36:38,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/3e0d858a59674344819abd941f1b66ec as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e0d858a59674344819abd941f1b66ec 2024-11-14T19:36:38,007 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e0d858a59674344819abd941f1b66ec, entries=150, sequenceid=251, filesize=11.9 K 2024-11-14T19:36:38,008 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d1f3fbc287ab8779d6c2fa31f746dbed in 1260ms, sequenceid=251, compaction requested=false 2024-11-14T19:36:38,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:38,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:38,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-14T19:36:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-14T19:36:38,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-14T19:36:38,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-14T19:36:38,014 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4190 sec 2024-11-14T19:36:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:38,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-14T19:36:38,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:38,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:38,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:38,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:38,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:38,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:38,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb58013570ce4b90ad83bae9d44ec528_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612998204/Put/seqid=0 2024-11-14T19:36:38,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742203_1379 (size=14994) 2024-11-14T19:36:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613058213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613058220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613058221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613058222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613058224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613058333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613058333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613058334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613058343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613058543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613058553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613058553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613058553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,616 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:38,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114eb58013570ce4b90ad83bae9d44ec528_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb58013570ce4b90ad83bae9d44ec528_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:38,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5652b170563143b1b20f3302f9675733, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:38,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5652b170563143b1b20f3302f9675733 is 175, key is test_row_0/A:col10/1731612998204/Put/seqid=0 2024-11-14T19:36:38,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742204_1380 (size=39949) 2024-11-14T19:36:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-14T19:36:38,743 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:38,743 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:38,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-14T19:36:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T19:36:38,745 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:38,746 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:38,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T19:36:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613058853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613058873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613058873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613058873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:38,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:38,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:38,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5652b170563143b1b20f3302f9675733 2024-11-14T19:36:39,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/5b89f5b43c364d6f818677f45b551f66 is 50, key is test_row_0/B:col10/1731612998204/Put/seqid=0 2024-11-14T19:36:39,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742205_1381 (size=12301) 2024-11-14T19:36:39,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T19:36:39,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613059233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:39,240 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in d1f3fbc287ab8779d6c2fa31f746dbed of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:36:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:39,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:39,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:39,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T19:36:39,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613059373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:39,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613059383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613059383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613059383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:39,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:39,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/5b89f5b43c364d6f818677f45b551f66 2024-11-14T19:36:39,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/93f2cf11ec3f403d9ad54358aefa2c95 is 50, key is test_row_0/C:col10/1731612998204/Put/seqid=0 2024-11-14T19:36:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742206_1382 (size=12301) 2024-11-14T19:36:39,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:39,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:39,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:39,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:39,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:39,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/93f2cf11ec3f403d9ad54358aefa2c95 2024-11-14T19:36:39,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/5652b170563143b1b20f3302f9675733 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5652b170563143b1b20f3302f9675733 2024-11-14T19:36:39,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5652b170563143b1b20f3302f9675733, entries=200, sequenceid=274, filesize=39.0 K 2024-11-14T19:36:39,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/5b89f5b43c364d6f818677f45b551f66 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/5b89f5b43c364d6f818677f45b551f66 2024-11-14T19:36:39,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/5b89f5b43c364d6f818677f45b551f66, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T19:36:39,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/93f2cf11ec3f403d9ad54358aefa2c95 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/93f2cf11ec3f403d9ad54358aefa2c95 2024-11-14T19:36:39,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/93f2cf11ec3f403d9ad54358aefa2c95, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T19:36:39,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d1f3fbc287ab8779d6c2fa31f746dbed in 1661ms, sequenceid=274, compaction requested=true 2024-11-14T19:36:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:39,866 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:39,866 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/A is initiating minor compaction (all files) 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/B is initiating minor compaction (all files) 2024-11-14T19:36:39,869 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/B in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,869 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/A in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,869 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/3b6ab989fa184353800c171254d0b81f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1307cad13fc248d0b7e358c087ab6ae5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5652b170563143b1b20f3302f9675733] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=100.3 K 2024-11-14T19:36:39,869 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1d29076ebae447db9e49247211dd3506, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1450b3f0250f43dcabc15c35c586fa07, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/5b89f5b43c364d6f818677f45b551f66] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=36.3 K 2024-11-14T19:36:39,869 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/3b6ab989fa184353800c171254d0b81f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1307cad13fc248d0b7e358c087ab6ae5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5652b170563143b1b20f3302f9675733] 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d29076ebae447db9e49247211dd3506, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731612994153 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b6ab989fa184353800c171254d0b81f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731612994153 2024-11-14T19:36:39,869 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1450b3f0250f43dcabc15c35c586fa07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731612994832 2024-11-14T19:36:39,870 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1307cad13fc248d0b7e358c087ab6ae5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731612994832 2024-11-14T19:36:39,870 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5652b170563143b1b20f3302f9675733, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731612997041 2024-11-14T19:36:39,870 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b89f5b43c364d6f818677f45b551f66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731612997041 2024-11-14T19:36:39,874 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:39,876 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114ba4641c372bc43659c8141105d0b0305_d1f3fbc287ab8779d6c2fa31f746dbed store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:39,877 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:39,877 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/29054984bd9c4d0b87b0df86abb34261 is 50, key is test_row_0/B:col10/1731612998204/Put/seqid=0 2024-11-14T19:36:39,878 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114ba4641c372bc43659c8141105d0b0305_d1f3fbc287ab8779d6c2fa31f746dbed, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:39,878 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ba4641c372bc43659c8141105d0b0305_d1f3fbc287ab8779d6c2fa31f746dbed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T19:36:39,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742208_1384 (size=4469) 2024-11-14T19:36:39,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742207_1383 (size=12949) 2024-11-14T19:36:39,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-14T19:36:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:39,968 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T19:36:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:39,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:39,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111483bd27bbe4eb42b5982eb079a0de1758_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731612998218/Put/seqid=0 2024-11-14T19:36:39,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742209_1385 (size=12454) 2024-11-14T19:36:40,287 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#A#compaction#314 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:40,287 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/678741b216cf443cba277e637c477e1f is 175, key is test_row_0/A:col10/1731612998204/Put/seqid=0 2024-11-14T19:36:40,292 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/29054984bd9c4d0b87b0df86abb34261 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/29054984bd9c4d0b87b0df86abb34261 2024-11-14T19:36:40,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742210_1386 (size=31903) 2024-11-14T19:36:40,296 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/B of d1f3fbc287ab8779d6c2fa31f746dbed into 29054984bd9c4d0b87b0df86abb34261(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:40,296 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:40,296 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/B, priority=13, startTime=1731612999866; duration=0sec 2024-11-14T19:36:40,296 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:40,296 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:B 2024-11-14T19:36:40,296 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:40,297 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:40,297 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/C is initiating minor compaction (all files) 2024-11-14T19:36:40,297 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/C in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:40,297 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f3c6f8e4d38499b9b3343ab9c4138d3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e0d858a59674344819abd941f1b66ec, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/93f2cf11ec3f403d9ad54358aefa2c95] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=36.3 K 2024-11-14T19:36:40,297 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f3c6f8e4d38499b9b3343ab9c4138d3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731612994153 2024-11-14T19:36:40,298 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0d858a59674344819abd941f1b66ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731612994832 2024-11-14T19:36:40,298 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f2cf11ec3f403d9ad54358aefa2c95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731612997041 2024-11-14T19:36:40,304 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:40,305 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/3e11b452d4fe440aaaa382a0541a905e is 50, key is test_row_0/C:col10/1731612998204/Put/seqid=0 2024-11-14T19:36:40,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742211_1387 (size=12949) 2024-11-14T19:36:40,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:40,386 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111483bd27bbe4eb42b5982eb079a0de1758_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111483bd27bbe4eb42b5982eb079a0de1758_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:40,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/8bcb731c98d948afaa1cfbe6ba281490, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:40,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/8bcb731c98d948afaa1cfbe6ba281490 is 175, key is test_row_0/A:col10/1731612998218/Put/seqid=0 2024-11-14T19:36:40,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. as already flushing 2024-11-14T19:36:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:40,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742212_1388 (size=31255) 2024-11-14T19:36:40,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731613000424, max timestamp=1731613000453Num uniques 859; 2024-11-14T19:36:40,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:36:40,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731613000424, max timestamp=1731613000453Num uniques 859; 2024-11-14T19:36:40,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:36:40,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731613000424, max timestamp=1731613000453Num uniques 859; 2024-11-14T19:36:40,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:36:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613060485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613060486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613060494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613060499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613060604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613060604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613060604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613060613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,697 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/678741b216cf443cba277e637c477e1f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/678741b216cf443cba277e637c477e1f 2024-11-14T19:36:40,701 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/A of d1f3fbc287ab8779d6c2fa31f746dbed into 678741b216cf443cba277e637c477e1f(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:40,701 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:40,701 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/A, priority=13, startTime=1731612999866; duration=0sec 2024-11-14T19:36:40,701 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:40,701 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:A 2024-11-14T19:36:40,724 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/3e11b452d4fe440aaaa382a0541a905e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e11b452d4fe440aaaa382a0541a905e 2024-11-14T19:36:40,728 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/C of d1f3fbc287ab8779d6c2fa31f746dbed into 3e11b452d4fe440aaaa382a0541a905e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:40,728 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:40,729 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/C, priority=13, startTime=1731612999866; duration=0sec 2024-11-14T19:36:40,729 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:40,729 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:C 2024-11-14T19:36:40,794 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/8bcb731c98d948afaa1cfbe6ba281490 2024-11-14T19:36:40,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/893f2682070141069db828c3a7110e8a is 50, key is test_row_0/B:col10/1731612998218/Put/seqid=0 2024-11-14T19:36:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742213_1389 (size=12301) 2024-11-14T19:36:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613060813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613060814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613060814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613060833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:40,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:40,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T19:36:41,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613061123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613061133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613061133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613061143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,204 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/893f2682070141069db828c3a7110e8a 2024-11-14T19:36:41,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f62e48284d8041c4b5f104a30cabe26b is 50, key is test_row_0/C:col10/1731612998218/Put/seqid=0 2024-11-14T19:36:41,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742214_1390 (size=12301) 2024-11-14T19:36:41,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36000 deadline: 1731613061253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,259 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in d1f3fbc287ab8779d6c2fa31f746dbed of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:36:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,616 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f62e48284d8041c4b5f104a30cabe26b 2024-11-14T19:36:41,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/8bcb731c98d948afaa1cfbe6ba281490 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8bcb731c98d948afaa1cfbe6ba281490 2024-11-14T19:36:41,623 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8bcb731c98d948afaa1cfbe6ba281490, entries=150, sequenceid=285, filesize=30.5 K 2024-11-14T19:36:41,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/893f2682070141069db828c3a7110e8a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/893f2682070141069db828c3a7110e8a 2024-11-14T19:36:41,627 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/893f2682070141069db828c3a7110e8a, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T19:36:41,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/f62e48284d8041c4b5f104a30cabe26b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f62e48284d8041c4b5f104a30cabe26b 2024-11-14T19:36:41,631 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f62e48284d8041c4b5f104a30cabe26b, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T19:36:41,632 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for d1f3fbc287ab8779d6c2fa31f746dbed in 1663ms, sequenceid=285, compaction requested=false 2024-11-14T19:36:41,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:41,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:41,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-14T19:36:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-14T19:36:41,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-14T19:36:41,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8870 sec 2024-11-14T19:36:41,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.8910 sec 2024-11-14T19:36:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:41,650 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-14T19:36:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:41,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c1a6f57b47a7474c897b716678ee30ad_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731613000493/Put/seqid=0 2024-11-14T19:36:41,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742215_1391 (size=14994) 2024-11-14T19:36:41,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613061664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613061665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613061669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613061677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613061783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613061783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613061783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613061793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,850 INFO [Thread-1425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1425. 2024-11-14T19:36:41,850 DEBUG [Thread-1425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:41,850 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,850 DEBUG [Thread-1425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,850 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:41,854 INFO [Thread-1429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1429. 2024-11-14T19:36:41,854 DEBUG [Thread-1429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:41,854 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,855 DEBUG [Thread-1429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:41,855 INFO [Thread-1423 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1423. 2024-11-14T19:36:41,856 DEBUG [Thread-1423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:41,856 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,856 DEBUG [Thread-1423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,856 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:41,862 INFO [Thread-1427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1427. 2024-11-14T19:36:41,862 DEBUG [Thread-1427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:41,862 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,862 DEBUG [Thread-1427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:41,866 INFO [Thread-1421 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1421. 2024-11-14T19:36:41,866 DEBUG [Thread-1421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:41,866 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,866 DEBUG [Thread-1421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:41,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:41,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613061993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613061993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613061994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613062003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,061 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:42,067 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c1a6f57b47a7474c897b716678ee30ad_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1a6f57b47a7474c897b716678ee30ad_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:42,068 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/15e3340cfe0b4b408d9b524e7e5a573a, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:42,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/15e3340cfe0b4b408d9b524e7e5a573a is 175, key is test_row_0/A:col10/1731613000493/Put/seqid=0 2024-11-14T19:36:42,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742216_1392 (size=39949) 2024-11-14T19:36:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613062303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613062303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613062303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613062313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/15e3340cfe0b4b408d9b524e7e5a573a 2024-11-14T19:36:42,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/c70f387e19f04d06b8b44f0b2aa60983 is 50, key is test_row_0/B:col10/1731613000493/Put/seqid=0 2024-11-14T19:36:42,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742217_1393 (size=12301) 2024-11-14T19:36:42,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35964 deadline: 1731613062814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35948 deadline: 1731613062814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35944 deadline: 1731613062815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35996 deadline: 1731613062824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:42,825 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:42,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:36:42,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/c70f387e19f04d06b8b44f0b2aa60983 2024-11-14T19:36:42,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/58d8a3113afc4aadb5ffa2f9f44643f7 is 50, key is test_row_0/C:col10/1731613000493/Put/seqid=0 2024-11-14T19:36:42,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742218_1394 (size=12301) 2024-11-14T19:36:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-14T19:36:42,913 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:43,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/58d8a3113afc4aadb5ffa2f9f44643f7 2024-11-14T19:36:43,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/15e3340cfe0b4b408d9b524e7e5a573a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/15e3340cfe0b4b408d9b524e7e5a573a 2024-11-14T19:36:43,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/15e3340cfe0b4b408d9b524e7e5a573a, entries=200, sequenceid=323, filesize=39.0 K 2024-11-14T19:36:43,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/c70f387e19f04d06b8b44f0b2aa60983 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c70f387e19f04d06b8b44f0b2aa60983 2024-11-14T19:36:43,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c70f387e19f04d06b8b44f0b2aa60983, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T19:36:43,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/58d8a3113afc4aadb5ffa2f9f44643f7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/58d8a3113afc4aadb5ffa2f9f44643f7 2024-11-14T19:36:43,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/58d8a3113afc4aadb5ffa2f9f44643f7, entries=150, sequenceid=323, filesize=12.0 K 2024-11-14T19:36:43,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for d1f3fbc287ab8779d6c2fa31f746dbed in 1687ms, sequenceid=323, compaction requested=true 2024-11-14T19:36:43,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:43,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:43,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:43,336 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:43,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:43,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:43,336 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:43,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d1f3fbc287ab8779d6c2fa31f746dbed:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:43,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:43,337 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:43,337 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/A is initiating minor compaction (all files) 2024-11-14T19:36:43,337 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:43,337 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/A in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:43,337 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/B is initiating minor compaction (all files) 2024-11-14T19:36:43,337 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/B in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:43,337 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/678741b216cf443cba277e637c477e1f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8bcb731c98d948afaa1cfbe6ba281490, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/15e3340cfe0b4b408d9b524e7e5a573a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=100.7 K 2024-11-14T19:36:43,337 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:43,337 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/29054984bd9c4d0b87b0df86abb34261, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/893f2682070141069db828c3a7110e8a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c70f387e19f04d06b8b44f0b2aa60983] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=36.7 K 2024-11-14T19:36:43,337 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/678741b216cf443cba277e637c477e1f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8bcb731c98d948afaa1cfbe6ba281490, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/15e3340cfe0b4b408d9b524e7e5a573a] 2024-11-14T19:36:43,338 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 29054984bd9c4d0b87b0df86abb34261, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731612997041 2024-11-14T19:36:43,338 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678741b216cf443cba277e637c477e1f, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731612997041 2024-11-14T19:36:43,338 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 893f2682070141069db828c3a7110e8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731612998218 2024-11-14T19:36:43,338 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcb731c98d948afaa1cfbe6ba281490, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731612998218 2024-11-14T19:36:43,338 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c70f387e19f04d06b8b44f0b2aa60983, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731613000493 2024-11-14T19:36:43,338 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e3340cfe0b4b408d9b524e7e5a573a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731613000468 2024-11-14T19:36:43,343 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:43,344 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#B#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:43,345 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114754b71feef9c40c6a7217b4f5ac96d9f_d1f3fbc287ab8779d6c2fa31f746dbed store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:43,345 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d465b59f7f2b460790852a46f57fcdcc is 50, key is test_row_0/B:col10/1731613000493/Put/seqid=0 2024-11-14T19:36:43,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742219_1395 (size=13051) 2024-11-14T19:36:43,354 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114754b71feef9c40c6a7217b4f5ac96d9f_d1f3fbc287ab8779d6c2fa31f746dbed, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:43,354 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114754b71feef9c40c6a7217b4f5ac96d9f_d1f3fbc287ab8779d6c2fa31f746dbed because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:43,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742220_1396 (size=4469) 2024-11-14T19:36:43,760 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:43,761 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e37766b934a84b4ca08ddc0979b2d736 is 175, key is test_row_0/A:col10/1731613000493/Put/seqid=0 2024-11-14T19:36:43,763 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/d465b59f7f2b460790852a46f57fcdcc as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d465b59f7f2b460790852a46f57fcdcc 2024-11-14T19:36:43,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742221_1397 (size=32005) 2024-11-14T19:36:43,768 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/B of d1f3fbc287ab8779d6c2fa31f746dbed into d465b59f7f2b460790852a46f57fcdcc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:43,768 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:43,768 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/B, priority=13, startTime=1731613003336; duration=0sec 2024-11-14T19:36:43,768 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:43,769 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:B 2024-11-14T19:36:43,769 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:43,769 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:43,769 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): d1f3fbc287ab8779d6c2fa31f746dbed/C is initiating minor compaction (all files) 2024-11-14T19:36:43,770 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d1f3fbc287ab8779d6c2fa31f746dbed/C in TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:43,770 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e11b452d4fe440aaaa382a0541a905e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f62e48284d8041c4b5f104a30cabe26b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/58d8a3113afc4aadb5ffa2f9f44643f7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp, totalSize=36.7 K 2024-11-14T19:36:43,770 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e11b452d4fe440aaaa382a0541a905e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731612997041 2024-11-14T19:36:43,770 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f62e48284d8041c4b5f104a30cabe26b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731612998218 2024-11-14T19:36:43,770 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 58d8a3113afc4aadb5ffa2f9f44643f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731613000493 2024-11-14T19:36:43,777 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d1f3fbc287ab8779d6c2fa31f746dbed#C#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:43,778 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/b6c95e089d1f4bd3aebb1fb9764a10d1 is 50, key is test_row_0/C:col10/1731613000493/Put/seqid=0 2024-11-14T19:36:43,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742222_1398 (size=13051) 2024-11-14T19:36:43,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:43,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T19:36:43,825 INFO [Thread-1414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1414. 2024-11-14T19:36:43,825 DEBUG [Thread-1414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:43,825 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,825 INFO [Thread-1416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1416. 2024-11-14T19:36:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:43,825 DEBUG [Thread-1416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:43,825 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,825 DEBUG [Thread-1414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:43,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:43,825 DEBUG [Thread-1416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:43,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:43,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e6a1ff0b10224f2cb50b4ed146d09481_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731613003823/Put/seqid=0 2024-11-14T19:36:43,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742223_1399 (size=12454) 2024-11-14T19:36:43,835 INFO [Thread-1412 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1412. 2024-11-14T19:36:43,835 DEBUG [Thread-1412 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:43,835 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,835 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:43,845 INFO [Thread-1410 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1410. 2024-11-14T19:36:43,845 DEBUG [Thread-1410 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:43,845 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,845 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:43,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:44,177 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/e37766b934a84b4ca08ddc0979b2d736 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e37766b934a84b4ca08ddc0979b2d736 2024-11-14T19:36:44,183 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/A of d1f3fbc287ab8779d6c2fa31f746dbed into e37766b934a84b4ca08ddc0979b2d736(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:44,183 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:44,183 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/A, priority=13, startTime=1731613003336; duration=0sec 2024-11-14T19:36:44,183 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:44,183 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:A 2024-11-14T19:36:44,185 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/b6c95e089d1f4bd3aebb1fb9764a10d1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b6c95e089d1f4bd3aebb1fb9764a10d1 2024-11-14T19:36:44,189 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d1f3fbc287ab8779d6c2fa31f746dbed/C of d1f3fbc287ab8779d6c2fa31f746dbed into b6c95e089d1f4bd3aebb1fb9764a10d1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:44,189 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:44,189 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., storeName=d1f3fbc287ab8779d6c2fa31f746dbed/C, priority=13, startTime=1731613003336; duration=0sec 2024-11-14T19:36:44,189 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:44,189 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d1f3fbc287ab8779d6c2fa31f746dbed:C 2024-11-14T19:36:44,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:44,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e6a1ff0b10224f2cb50b4ed146d09481_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e6a1ff0b10224f2cb50b4ed146d09481_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:44,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/72251b612089491da42fa8bf4e0afb36, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:44,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/72251b612089491da42fa8bf4e0afb36 is 175, key is test_row_0/A:col10/1731613003823/Put/seqid=0 2024-11-14T19:36:44,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742224_1400 (size=31255) 2024-11-14T19:36:44,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/72251b612089491da42fa8bf4e0afb36 2024-11-14T19:36:44,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b44704450a5a4582b9d9a3a1464c0a68 is 50, key is test_row_0/B:col10/1731613003823/Put/seqid=0 2024-11-14T19:36:44,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742225_1401 (size=12301) 2024-11-14T19:36:45,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b44704450a5a4582b9d9a3a1464c0a68 2024-11-14T19:36:45,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/37174665134b4618a78d534365fc3354 is 50, key is test_row_0/C:col10/1731613003823/Put/seqid=0 2024-11-14T19:36:45,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742226_1402 (size=12301) 2024-11-14T19:36:45,285 INFO [Thread-1418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1418. 2024-11-14T19:36:45,285 DEBUG [Thread-1418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:36:45,285 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:45,285 DEBUG [Thread-1418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:45,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1031 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3071 rows 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1025 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3075 rows 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1011 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3033 rows 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1042 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3074 rows 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1034 2024-11-14T19:36:45,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3069 rows 2024-11-14T19:36:45,285 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:36:45,286 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:36:45,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:45,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:45,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:36:45,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T19:36:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:45,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T19:36:45,289 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613005289"}]},"ts":"1731613005289"} 2024-11-14T19:36:45,291 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T19:36:45,291 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T19:36:45,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:36:45,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, UNASSIGN}] 2024-11-14T19:36:45,294 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, UNASSIGN 2024-11-14T19:36:45,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:45,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, UNASSIGN because future has completed 2024-11-14T19:36:45,297 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:36:45,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:36:45,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T19:36:45,450 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:45,450 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:36:45,450 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing d1f3fbc287ab8779d6c2fa31f746dbed, disabling compactions & flushes 2024-11-14T19:36:45,450 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:45,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/37174665134b4618a78d534365fc3354 2024-11-14T19:36:45,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/72251b612089491da42fa8bf4e0afb36 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/72251b612089491da42fa8bf4e0afb36 2024-11-14T19:36:45,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/72251b612089491da42fa8bf4e0afb36, entries=150, sequenceid=336, filesize=30.5 K 2024-11-14T19:36:45,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/b44704450a5a4582b9d9a3a1464c0a68 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b44704450a5a4582b9d9a3a1464c0a68 2024-11-14T19:36:45,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b44704450a5a4582b9d9a3a1464c0a68, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T19:36:45,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/37174665134b4618a78d534365fc3354 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/37174665134b4618a78d534365fc3354 2024-11-14T19:36:45,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/37174665134b4618a78d534365fc3354, entries=150, sequenceid=336, filesize=12.0 K 2024-11-14T19:36:45,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for d1f3fbc287ab8779d6c2fa31f746dbed in 1674ms, sequenceid=336, compaction requested=false 2024-11-14T19:36:45,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d1f3fbc287ab8779d6c2fa31f746dbed: 2024-11-14T19:36:45,498 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:45,498 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:45,498 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. after waiting 0 ms 2024-11-14T19:36:45,498 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:45,498 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing d1f3fbc287ab8779d6c2fa31f746dbed 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-14T19:36:45,499 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=A 2024-11-14T19:36:45,499 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:45,499 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=B 2024-11-14T19:36:45,499 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:45,499 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d1f3fbc287ab8779d6c2fa31f746dbed, store=C 2024-11-14T19:36:45,499 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:45,504 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ccc2b8ec14149b2a79532e64c1249c0_d1f3fbc287ab8779d6c2fa31f746dbed is 50, key is test_row_0/A:col10/1731613005283/Put/seqid=0 2024-11-14T19:36:45,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742227_1403 (size=9914) 2024-11-14T19:36:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T19:36:45,912 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:45,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T19:36:45,917 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149ccc2b8ec14149b2a79532e64c1249c0_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ccc2b8ec14149b2a79532e64c1249c0_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:45,917 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b6d4b7be3bc24937a43ba9b69ac540fe, store: [table=TestAcidGuarantees family=A region=d1f3fbc287ab8779d6c2fa31f746dbed] 2024-11-14T19:36:45,918 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b6d4b7be3bc24937a43ba9b69ac540fe is 175, key is test_row_0/A:col10/1731613005283/Put/seqid=0 2024-11-14T19:36:45,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742228_1404 (size=22561) 2024-11-14T19:36:46,323 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b6d4b7be3bc24937a43ba9b69ac540fe 2024-11-14T19:36:46,332 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/0e43cce9195741568a397a0188199e3f is 50, key is test_row_0/B:col10/1731613005283/Put/seqid=0 2024-11-14T19:36:46,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742229_1405 (size=9857) 2024-11-14T19:36:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T19:36:46,736 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/0e43cce9195741568a397a0188199e3f 2024-11-14T19:36:46,742 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/fbea5e0d497a404e972748662d4c2442 is 50, key is test_row_0/C:col10/1731613005283/Put/seqid=0 2024-11-14T19:36:46,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742230_1406 (size=9857) 2024-11-14T19:36:46,861 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T19:36:47,147 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/fbea5e0d497a404e972748662d4c2442 2024-11-14T19:36:47,152 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/A/b6d4b7be3bc24937a43ba9b69ac540fe as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b6d4b7be3bc24937a43ba9b69ac540fe 2024-11-14T19:36:47,156 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b6d4b7be3bc24937a43ba9b69ac540fe, entries=100, sequenceid=344, filesize=22.0 K 2024-11-14T19:36:47,156 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/B/0e43cce9195741568a397a0188199e3f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/0e43cce9195741568a397a0188199e3f 2024-11-14T19:36:47,160 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/0e43cce9195741568a397a0188199e3f, entries=100, sequenceid=344, filesize=9.6 K 2024-11-14T19:36:47,161 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/.tmp/C/fbea5e0d497a404e972748662d4c2442 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/fbea5e0d497a404e972748662d4c2442 2024-11-14T19:36:47,164 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/fbea5e0d497a404e972748662d4c2442, entries=100, sequenceid=344, filesize=9.6 K 2024-11-14T19:36:47,165 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d1f3fbc287ab8779d6c2fa31f746dbed in 1667ms, sequenceid=344, compaction requested=true 2024-11-14T19:36:47,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/4280638f6ed64fe18a92f50f9e84e5a5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/46ead1675b4d4c8388ad08777e654f0c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/94330c59b53c4db2bb71a00ddf569c28, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/572c2ca3000843a1aafefc8ba008fe52, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5da5cbb9da824aec9623e7a6d24f7440, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e33713cade0249859d50823d51215ce5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/a6378904fc0742bc9cd0dc64cc4d8b79, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b9bd72cdbb6a499f8ffa355cbd2696d3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1e5757fb185d4a98b71b8e5b52660f13, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7fc1cade71484ea08be935f08caecd72, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7424eed8634144d2912ebaa9e89e3eb5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8ad01ecf10ba47a5a638391ad63d6473, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/06f47d47caea44cbbfbaa51f533178f8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/10e5376bb3364384a199b65722841bd1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e522f3620279409b86b3d1c1184f65dd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/3b6ab989fa184353800c171254d0b81f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1307cad13fc248d0b7e358c087ab6ae5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5652b170563143b1b20f3302f9675733, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/678741b216cf443cba277e637c477e1f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8bcb731c98d948afaa1cfbe6ba281490, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/15e3340cfe0b4b408d9b524e7e5a573a] to archive 2024-11-14T19:36:47,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:47,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/17c2b17c5f4a41e9b1a21fd6eb9b7c7c 2024-11-14T19:36:47,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/4280638f6ed64fe18a92f50f9e84e5a5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/4280638f6ed64fe18a92f50f9e84e5a5 2024-11-14T19:36:47,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/46ead1675b4d4c8388ad08777e654f0c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/46ead1675b4d4c8388ad08777e654f0c 2024-11-14T19:36:47,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/94330c59b53c4db2bb71a00ddf569c28 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/94330c59b53c4db2bb71a00ddf569c28 2024-11-14T19:36:47,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/572c2ca3000843a1aafefc8ba008fe52 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/572c2ca3000843a1aafefc8ba008fe52 2024-11-14T19:36:47,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5da5cbb9da824aec9623e7a6d24f7440 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5da5cbb9da824aec9623e7a6d24f7440 2024-11-14T19:36:47,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e33713cade0249859d50823d51215ce5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e33713cade0249859d50823d51215ce5 2024-11-14T19:36:47,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/a6378904fc0742bc9cd0dc64cc4d8b79 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/a6378904fc0742bc9cd0dc64cc4d8b79 2024-11-14T19:36:47,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b9bd72cdbb6a499f8ffa355cbd2696d3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b9bd72cdbb6a499f8ffa355cbd2696d3 2024-11-14T19:36:47,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1e5757fb185d4a98b71b8e5b52660f13 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1e5757fb185d4a98b71b8e5b52660f13 2024-11-14T19:36:47,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7fc1cade71484ea08be935f08caecd72 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7fc1cade71484ea08be935f08caecd72 2024-11-14T19:36:47,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7424eed8634144d2912ebaa9e89e3eb5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/7424eed8634144d2912ebaa9e89e3eb5 2024-11-14T19:36:47,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8ad01ecf10ba47a5a638391ad63d6473 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8ad01ecf10ba47a5a638391ad63d6473 2024-11-14T19:36:47,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/06f47d47caea44cbbfbaa51f533178f8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/06f47d47caea44cbbfbaa51f533178f8 2024-11-14T19:36:47,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/10e5376bb3364384a199b65722841bd1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/10e5376bb3364384a199b65722841bd1 2024-11-14T19:36:47,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e522f3620279409b86b3d1c1184f65dd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e522f3620279409b86b3d1c1184f65dd 2024-11-14T19:36:47,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/3b6ab989fa184353800c171254d0b81f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/3b6ab989fa184353800c171254d0b81f 2024-11-14T19:36:47,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1307cad13fc248d0b7e358c087ab6ae5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/1307cad13fc248d0b7e358c087ab6ae5 2024-11-14T19:36:47,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5652b170563143b1b20f3302f9675733 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/5652b170563143b1b20f3302f9675733 2024-11-14T19:36:47,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/678741b216cf443cba277e637c477e1f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/678741b216cf443cba277e637c477e1f 2024-11-14T19:36:47,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8bcb731c98d948afaa1cfbe6ba281490 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/8bcb731c98d948afaa1cfbe6ba281490 2024-11-14T19:36:47,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/15e3340cfe0b4b408d9b524e7e5a573a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/15e3340cfe0b4b408d9b524e7e5a573a 2024-11-14T19:36:47,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d18eed1e86474b6f82a7bfbdec9dcf66, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/e355242f022f4c86b663bb66ef7f217b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/7e2ee6e99eab4b6099cec3933f503e2a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/56dfa86db265496d85eb7a82667f60fb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c762a943683c4ba9a591313d5f39d6e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/3af65acc8aee498984ee469d85e49e13, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d804f7611a1b4113bd10340749cc48cf, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/16928a442cad48538ad62cfc840be0d5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/8c4b2ce67fad424d9574f918a630286c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/fd00712015554ffa9e46cd6676381020, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b2f9f9bd801449d5a3511119abd9ee2c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/92c15476a44a448399533d3aa495ce45, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b722a86c58af431b824ed251e8f907a7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/4bde6a14fb034a179756412a2d7fc920, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1d29076ebae447db9e49247211dd3506, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/21dad3ca437a489cb0c692610cfa3f27, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1450b3f0250f43dcabc15c35c586fa07, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/29054984bd9c4d0b87b0df86abb34261, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/5b89f5b43c364d6f818677f45b551f66, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/893f2682070141069db828c3a7110e8a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c70f387e19f04d06b8b44f0b2aa60983] to archive 2024-11-14T19:36:47,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:47,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d18eed1e86474b6f82a7bfbdec9dcf66 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d18eed1e86474b6f82a7bfbdec9dcf66 2024-11-14T19:36:47,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/e355242f022f4c86b663bb66ef7f217b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/e355242f022f4c86b663bb66ef7f217b 2024-11-14T19:36:47,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/7e2ee6e99eab4b6099cec3933f503e2a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/7e2ee6e99eab4b6099cec3933f503e2a 2024-11-14T19:36:47,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/56dfa86db265496d85eb7a82667f60fb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/56dfa86db265496d85eb7a82667f60fb 2024-11-14T19:36:47,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c762a943683c4ba9a591313d5f39d6e6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c762a943683c4ba9a591313d5f39d6e6 2024-11-14T19:36:47,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/3af65acc8aee498984ee469d85e49e13 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/3af65acc8aee498984ee469d85e49e13 2024-11-14T19:36:47,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d804f7611a1b4113bd10340749cc48cf to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d804f7611a1b4113bd10340749cc48cf 2024-11-14T19:36:47,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/16928a442cad48538ad62cfc840be0d5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/16928a442cad48538ad62cfc840be0d5 2024-11-14T19:36:47,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/8c4b2ce67fad424d9574f918a630286c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/8c4b2ce67fad424d9574f918a630286c 2024-11-14T19:36:47,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/fd00712015554ffa9e46cd6676381020 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/fd00712015554ffa9e46cd6676381020 2024-11-14T19:36:47,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b2f9f9bd801449d5a3511119abd9ee2c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b2f9f9bd801449d5a3511119abd9ee2c 2024-11-14T19:36:47,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/92c15476a44a448399533d3aa495ce45 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/92c15476a44a448399533d3aa495ce45 2024-11-14T19:36:47,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b722a86c58af431b824ed251e8f907a7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b722a86c58af431b824ed251e8f907a7 2024-11-14T19:36:47,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/6a7ed5f5e4c64a3aa5e10c2509d0cffa 2024-11-14T19:36:47,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/4bde6a14fb034a179756412a2d7fc920 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/4bde6a14fb034a179756412a2d7fc920 2024-11-14T19:36:47,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1d29076ebae447db9e49247211dd3506 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1d29076ebae447db9e49247211dd3506 2024-11-14T19:36:47,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/21dad3ca437a489cb0c692610cfa3f27 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/21dad3ca437a489cb0c692610cfa3f27 2024-11-14T19:36:47,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1450b3f0250f43dcabc15c35c586fa07 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/1450b3f0250f43dcabc15c35c586fa07 2024-11-14T19:36:47,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/29054984bd9c4d0b87b0df86abb34261 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/29054984bd9c4d0b87b0df86abb34261 2024-11-14T19:36:47,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/5b89f5b43c364d6f818677f45b551f66 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/5b89f5b43c364d6f818677f45b551f66 2024-11-14T19:36:47,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/893f2682070141069db828c3a7110e8a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/893f2682070141069db828c3a7110e8a 2024-11-14T19:36:47,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c70f387e19f04d06b8b44f0b2aa60983 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/c70f387e19f04d06b8b44f0b2aa60983 2024-11-14T19:36:47,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/034c9f3ac5ec46979976dd2c04f051d2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f902fae6ce7848e8a0c44918c625b552, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f24f4454e91d413faf520096048c9d27, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cb89ae19a12d4320966ff90399ef9c29, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b5bfa314c31b4035a68949b0208e21a0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/0a789d02fb144bcda12be8f34a03bee2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f2be06f67be40b1b7467ee025d8d448, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cea4175101454e67920ca0e8c8b73c03, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/65fe551f2f4348198902eb1392bb1f44, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/94d5131b8bd9492e8af61ca0a825a0c1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/e0a6d9de59b24d58ae02b6ba49af1eda, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/528e2d50baff446794aa91fda77e2ff8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/958a530de1a4490498930d0c79c893a6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/4217b49e520f498c8ad9cd6368cb91b3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f3c6f8e4d38499b9b3343ab9c4138d3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/576d9d63c1604503b62bc2d86a0e9d16, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e0d858a59674344819abd941f1b66ec, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e11b452d4fe440aaaa382a0541a905e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/93f2cf11ec3f403d9ad54358aefa2c95, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f62e48284d8041c4b5f104a30cabe26b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/58d8a3113afc4aadb5ffa2f9f44643f7] to archive 2024-11-14T19:36:47,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:47,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/034c9f3ac5ec46979976dd2c04f051d2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/034c9f3ac5ec46979976dd2c04f051d2 2024-11-14T19:36:47,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f902fae6ce7848e8a0c44918c625b552 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f902fae6ce7848e8a0c44918c625b552 2024-11-14T19:36:47,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f24f4454e91d413faf520096048c9d27 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f24f4454e91d413faf520096048c9d27 2024-11-14T19:36:47,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cb89ae19a12d4320966ff90399ef9c29 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cb89ae19a12d4320966ff90399ef9c29 2024-11-14T19:36:47,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/ed8f7ab5fafc48fcb33fec0f0f23d6c9 2024-11-14T19:36:47,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b5bfa314c31b4035a68949b0208e21a0 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b5bfa314c31b4035a68949b0208e21a0 2024-11-14T19:36:47,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/0a789d02fb144bcda12be8f34a03bee2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/0a789d02fb144bcda12be8f34a03bee2 2024-11-14T19:36:47,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f2be06f67be40b1b7467ee025d8d448 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f2be06f67be40b1b7467ee025d8d448 2024-11-14T19:36:47,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cea4175101454e67920ca0e8c8b73c03 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/cea4175101454e67920ca0e8c8b73c03 2024-11-14T19:36:47,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/65fe551f2f4348198902eb1392bb1f44 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/65fe551f2f4348198902eb1392bb1f44 2024-11-14T19:36:47,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/94d5131b8bd9492e8af61ca0a825a0c1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/94d5131b8bd9492e8af61ca0a825a0c1 2024-11-14T19:36:47,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/e0a6d9de59b24d58ae02b6ba49af1eda to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/e0a6d9de59b24d58ae02b6ba49af1eda 2024-11-14T19:36:47,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/528e2d50baff446794aa91fda77e2ff8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/528e2d50baff446794aa91fda77e2ff8 2024-11-14T19:36:47,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/958a530de1a4490498930d0c79c893a6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/958a530de1a4490498930d0c79c893a6 2024-11-14T19:36:47,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/4217b49e520f498c8ad9cd6368cb91b3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/4217b49e520f498c8ad9cd6368cb91b3 2024-11-14T19:36:47,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f3c6f8e4d38499b9b3343ab9c4138d3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/7f3c6f8e4d38499b9b3343ab9c4138d3 2024-11-14T19:36:47,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/576d9d63c1604503b62bc2d86a0e9d16 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/576d9d63c1604503b62bc2d86a0e9d16 2024-11-14T19:36:47,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e0d858a59674344819abd941f1b66ec to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e0d858a59674344819abd941f1b66ec 2024-11-14T19:36:47,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e11b452d4fe440aaaa382a0541a905e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/3e11b452d4fe440aaaa382a0541a905e 2024-11-14T19:36:47,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/93f2cf11ec3f403d9ad54358aefa2c95 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/93f2cf11ec3f403d9ad54358aefa2c95 2024-11-14T19:36:47,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f62e48284d8041c4b5f104a30cabe26b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/f62e48284d8041c4b5f104a30cabe26b 2024-11-14T19:36:47,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/58d8a3113afc4aadb5ffa2f9f44643f7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/58d8a3113afc4aadb5ffa2f9f44643f7 2024-11-14T19:36:47,235 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/recovered.edits/347.seqid, newMaxSeqId=347, maxSeqId=4 2024-11-14T19:36:47,235 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. 2024-11-14T19:36:47,235 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for d1f3fbc287ab8779d6c2fa31f746dbed: Waiting for close lock at 1731613005450Running coprocessor pre-close hooks at 1731613005450Disabling compacts and flushes for region at 1731613005450Disabling writes for close at 1731613005498 (+48 ms)Obtaining lock to block concurrent updates at 1731613005498Preparing flush snapshotting stores in d1f3fbc287ab8779d6c2fa31f746dbed at 1731613005498Finished memstore snapshotting TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731613005499 (+1 ms)Flushing stores of TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed. at 1731613005499Flushing d1f3fbc287ab8779d6c2fa31f746dbed/A: creating writer at 1731613005499Flushing mob file d1f3fbc287ab8779d6c2fa31f746dbed/A: appending metadata at 1731613005504 (+5 ms)Flushing mob file d1f3fbc287ab8779d6c2fa31f746dbed/A: closing flushed file at 1731613005504Flushing d1f3fbc287ab8779d6c2fa31f746dbed/A: appending metadata at 1731613005917 (+413 ms)Flushing d1f3fbc287ab8779d6c2fa31f746dbed/A: closing flushed file at 1731613005918 (+1 ms)Flushing d1f3fbc287ab8779d6c2fa31f746dbed/B: creating writer at 1731613006326 (+408 ms)Flushing d1f3fbc287ab8779d6c2fa31f746dbed/B: appending metadata at 1731613006331 (+5 ms)Flushing d1f3fbc287ab8779d6c2fa31f746dbed/B: closing flushed file at 1731613006331Flushing d1f3fbc287ab8779d6c2fa31f746dbed/C: creating writer at 1731613006739 (+408 ms)Flushing d1f3fbc287ab8779d6c2fa31f746dbed/C: appending metadata at 1731613006742 (+3 ms)Flushing d1f3fbc287ab8779d6c2fa31f746dbed/C: closing flushed file at 1731613006742Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7552eb7e: reopening flushed file at 1731613007151 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c493261: reopening flushed file at 1731613007156 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7521f564: reopening flushed file at 1731613007160 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d1f3fbc287ab8779d6c2fa31f746dbed in 1667ms, sequenceid=344, compaction requested=true at 1731613007165 (+5 ms)Writing region close event to WAL at 1731613007232 (+67 ms)Running coprocessor post-close hooks at 1731613007235 (+3 ms)Closed at 1731613007235 2024-11-14T19:36:47,237 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,238 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=d1f3fbc287ab8779d6c2fa31f746dbed, regionState=CLOSED 2024-11-14T19:36:47,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:36:47,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-14T19:36:47,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure d1f3fbc287ab8779d6c2fa31f746dbed, server=867b237d0fa7,38601,1731612889637 in 1.9440 sec 2024-11-14T19:36:47,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-14T19:36:47,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d1f3fbc287ab8779d6c2fa31f746dbed, UNASSIGN in 1.9490 sec 2024-11-14T19:36:47,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-14T19:36:47,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9520 sec 2024-11-14T19:36:47,247 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613007247"}]},"ts":"1731613007247"} 2024-11-14T19:36:47,248 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T19:36:47,248 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T19:36:47,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9630 sec 2024-11-14T19:36:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-14T19:36:47,444 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:47,444 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T19:36:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:47,446 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T19:36:47,446 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:47,448 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,450 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/recovered.edits] 2024-11-14T19:36:47,452 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/72251b612089491da42fa8bf4e0afb36 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/72251b612089491da42fa8bf4e0afb36 2024-11-14T19:36:47,453 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b6d4b7be3bc24937a43ba9b69ac540fe to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/b6d4b7be3bc24937a43ba9b69ac540fe 2024-11-14T19:36:47,454 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e37766b934a84b4ca08ddc0979b2d736 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/A/e37766b934a84b4ca08ddc0979b2d736 2024-11-14T19:36:47,456 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/0e43cce9195741568a397a0188199e3f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/0e43cce9195741568a397a0188199e3f 2024-11-14T19:36:47,457 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b44704450a5a4582b9d9a3a1464c0a68 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/b44704450a5a4582b9d9a3a1464c0a68 2024-11-14T19:36:47,458 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d465b59f7f2b460790852a46f57fcdcc to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/B/d465b59f7f2b460790852a46f57fcdcc 2024-11-14T19:36:47,464 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/37174665134b4618a78d534365fc3354 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/37174665134b4618a78d534365fc3354 2024-11-14T19:36:47,465 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b6c95e089d1f4bd3aebb1fb9764a10d1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/b6c95e089d1f4bd3aebb1fb9764a10d1 2024-11-14T19:36:47,466 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/fbea5e0d497a404e972748662d4c2442 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/C/fbea5e0d497a404e972748662d4c2442 2024-11-14T19:36:47,468 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/recovered.edits/347.seqid to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed/recovered.edits/347.seqid 2024-11-14T19:36:47,469 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,469 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T19:36:47,469 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T19:36:47,470 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T19:36:47,472 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114095f49a32f2741d6ac358a070d82b413_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114095f49a32f2741d6ac358a070d82b413_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,473 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141b1fdb066c2d4866ac5bcd8d6bd4a460_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141b1fdb066c2d4866ac5bcd8d6bd4a460_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,474 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142476aeddcc9a42de93b95f7e8722b88f_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411142476aeddcc9a42de93b95f7e8722b88f_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,474 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114301560294e9a48c3b5ecc8227b2beb17_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114301560294e9a48c3b5ecc8227b2beb17_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,475 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143760a41f5d0447fe8ae07a58611794ef_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143760a41f5d0447fe8ae07a58611794ef_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,476 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144880cdd3f1bd44b7945053da522759fc_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144880cdd3f1bd44b7945053da522759fc_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,477 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146182c0f9d1ef43d79f92bf0352f1f0cc_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411146182c0f9d1ef43d79f92bf0352f1f0cc_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,478 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465ca6c7443934a85a94cdf643624269a_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111465ca6c7443934a85a94cdf643624269a_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,479 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111483bd27bbe4eb42b5982eb079a0de1758_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111483bd27bbe4eb42b5982eb079a0de1758_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,480 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ccc2b8ec14149b2a79532e64c1249c0_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149ccc2b8ec14149b2a79532e64c1249c0_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,480 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3c1a105097e45debd6c42b8581ade06_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a3c1a105097e45debd6c42b8581ade06_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,481 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1a6f57b47a7474c897b716678ee30ad_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c1a6f57b47a7474c897b716678ee30ad_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,482 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3b5d81b4e694249a784ff00c60b3a82_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c3b5d81b4e694249a784ff00c60b3a82_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,483 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e6a1ff0b10224f2cb50b4ed146d09481_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e6a1ff0b10224f2cb50b4ed146d09481_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,483 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e83c0453dbed406e9defbd1c6d6728ae_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e83c0453dbed406e9defbd1c6d6728ae_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,484 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb0fc3f1f0be4094b57ec0b78a60d8cf_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb0fc3f1f0be4094b57ec0b78a60d8cf_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,485 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb58013570ce4b90ad83bae9d44ec528_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114eb58013570ce4b90ad83bae9d44ec528_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,486 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f90e150ad965428d83ba24eba60f4d60_d1f3fbc287ab8779d6c2fa31f746dbed to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114f90e150ad965428d83ba24eba60f4d60_d1f3fbc287ab8779d6c2fa31f746dbed 2024-11-14T19:36:47,487 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T19:36:47,489 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:47,491 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T19:36:47,493 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T19:36:47,494 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:47,494 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T19:36:47,494 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731613007494"}]},"ts":"9223372036854775807"} 2024-11-14T19:36:47,496 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T19:36:47,496 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d1f3fbc287ab8779d6c2fa31f746dbed, NAME => 'TestAcidGuarantees,,1731612978574.d1f3fbc287ab8779d6c2fa31f746dbed.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T19:36:47,496 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T19:36:47,496 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731613007496"}]},"ts":"9223372036854775807"} 2024-11-14T19:36:47,498 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T19:36:47,498 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:47,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-14T19:36:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-14T19:36:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T19:36:47,554 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:47,564 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 250), OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 576), ProcessCount=11 (was 11), AvailableMemoryMB=5498 (was 5577) 2024-11-14T19:36:47,574 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=572, ProcessCount=11, AvailableMemoryMB=5498 2024-11-14T19:36:47,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:36:47,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:36:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T19:36:47,578 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T19:36:47,578 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:47,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-14T19:36:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T19:36:47,579 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T19:36:47,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742231_1407 (size=1061) 2024-11-14T19:36:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T19:36:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T19:36:47,985 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:36:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742232_1408 (size=53) 2024-11-14T19:36:47,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:36:47,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae3a414a536a107c2204f2532a580772, disabling compactions & flushes 2024-11-14T19:36:47,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:47,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:47,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. after waiting 0 ms 2024-11-14T19:36:47,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:47,992 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:47,992 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae3a414a536a107c2204f2532a580772: Waiting for close lock at 1731613007992Disabling compacts and flushes for region at 1731613007992Disabling writes for close at 1731613007992Writing region close event to WAL at 1731613007992Closed at 1731613007992 2024-11-14T19:36:47,993 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T19:36:47,994 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731613007994"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731613007994"}]},"ts":"1731613007994"} 2024-11-14T19:36:47,995 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T19:36:47,996 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T19:36:47,996 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613007996"}]},"ts":"1731613007996"} 2024-11-14T19:36:47,997 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T19:36:47,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, ASSIGN}] 2024-11-14T19:36:47,998 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, ASSIGN 2024-11-14T19:36:47,999 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, ASSIGN; state=OFFLINE, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=false 2024-11-14T19:36:48,149 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=ae3a414a536a107c2204f2532a580772, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:48,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, ASSIGN because future has completed 2024-11-14T19:36:48,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:36:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T19:36:48,304 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:48,305 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:36:48,305 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,305 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:36:48,305 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,305 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,306 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,307 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:48,308 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3a414a536a107c2204f2532a580772 columnFamilyName A 2024-11-14T19:36:48,308 DEBUG [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:48,308 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.HStore(327): Store=ae3a414a536a107c2204f2532a580772/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:48,308 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,309 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:48,309 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3a414a536a107c2204f2532a580772 columnFamilyName B 2024-11-14T19:36:48,310 DEBUG [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:48,310 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.HStore(327): Store=ae3a414a536a107c2204f2532a580772/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:48,310 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,311 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:36:48,311 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3a414a536a107c2204f2532a580772 columnFamilyName C 2024-11-14T19:36:48,311 DEBUG [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:36:48,311 INFO [StoreOpener-ae3a414a536a107c2204f2532a580772-1 {}] regionserver.HStore(327): Store=ae3a414a536a107c2204f2532a580772/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:36:48,311 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:48,312 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,312 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,312 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,313 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,313 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,314 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:36:48,315 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,316 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:36:48,317 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened ae3a414a536a107c2204f2532a580772; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65042258, jitterRate=-0.03079482913017273}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:36:48,317 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,318 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for ae3a414a536a107c2204f2532a580772: Running coprocessor pre-open hook at 1731613008305Writing region info on filesystem at 1731613008305Initializing all the Stores at 1731613008306 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613008306Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613008306Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613008306Cleaning up temporary data from old regions at 1731613008313 (+7 ms)Running coprocessor post-open hooks at 1731613008317 (+4 ms)Region opened successfully at 1731613008318 (+1 ms) 2024-11-14T19:36:48,318 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., pid=114, masterSystemTime=1731613008302 2024-11-14T19:36:48,320 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:48,320 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:48,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=ae3a414a536a107c2204f2532a580772, regionState=OPEN, openSeqNum=2, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:48,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:36:48,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-14T19:36:48,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 in 171 msec 2024-11-14T19:36:48,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-14T19:36:48,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, ASSIGN in 326 msec 2024-11-14T19:36:48,325 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T19:36:48,325 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613008325"}]},"ts":"1731613008325"} 2024-11-14T19:36:48,327 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T19:36:48,327 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T19:36:48,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 751 msec 2024-11-14T19:36:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-14T19:36:48,713 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:48,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68f36482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,715 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,715 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,716 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c47293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,717 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,718 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1303523c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,718 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T19:36:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 867b237d0fa7,38019,1731612888274 2024-11-14T19:36:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32581898 2024-11-14T19:36:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T19:36:48,722 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T19:36:48,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bfd8f06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,724 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ccf5df4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b2caa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dee51f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26201a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c77f34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d395438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671b25f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40440054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d6985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,739 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c065b0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736f75a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490627bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d1acc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14508764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4667e0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,745 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3344f5e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,746 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@697aeffd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794001fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,748 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,748 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dcfa564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28595a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3c4fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4745c5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fdce2b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b24ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,757 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5653e707, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,758 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23cef71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaa8538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:36:48,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:36:48,760 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c53b38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:36:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:36:48,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:36:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e407dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:36:48,767 DEBUG [Thread-1862 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,779 DEBUG [Thread-1845 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,781 DEBUG [Thread-1849 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,787 DEBUG [Thread-1864 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,788 DEBUG [Thread-1858 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,791 DEBUG [Thread-1853 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,792 DEBUG [Thread-1853 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,793 DEBUG [Thread-1853 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,795 DEBUG [Thread-1856 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,795 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:48,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,796 DEBUG [Thread-1856 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,796 DEBUG [Thread-1856 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,798 DEBUG [Thread-1851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-14T19:36:48,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,799 DEBUG [Thread-1847 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,799 DEBUG [Thread-1851 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T19:36:48,799 DEBUG [Thread-1851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,799 DEBUG [Thread-1847 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,800 DEBUG [Thread-1860 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:36:48,801 DEBUG [Thread-1853 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,801 DEBUG [Thread-1847 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:36:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:36:48,803 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:36:48,806 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:48,807 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:48,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:48,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,811 DEBUG [Thread-1851 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2] 2024-11-14T19:36:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:48,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T19:36:48,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:48,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:48,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/5f82da6fe0594c239af09d4be14e19c3 is 50, key is test_row_0/A:col10/1731613008801/Put/seqid=0 2024-11-14T19:36:48,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742233_1409 (size=16681) 2024-11-14T19:36:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731613008854, max timestamp=1731613008883Num uniques 859; 2024-11-14T19:36:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731613008854, max timestamp=1731613008883Num uniques 859; 2024-11-14T19:36:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:36:48,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:36:48,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731613008854, max timestamp=1731613008883Num uniques 859; 2024-11-14T19:36:48,888 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:36:48,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T19:36:48,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613068932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:48,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613068936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:48,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:48,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:48,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:48,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613068940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:48,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613068950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:48,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613068950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:48,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613069053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613069063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613069063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613069073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613069073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T19:36:49,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/5f82da6fe0594c239af09d4be14e19c3 2024-11-14T19:36:49,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613069273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613069283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613069284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613069296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613069296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/948bed12ae7448deb6d316c2ea5a46a0 is 50, key is test_row_0/B:col10/1731613008801/Put/seqid=0 2024-11-14T19:36:49,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742234_1410 (size=12001) 2024-11-14T19:36:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T19:36:49,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:49,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:49,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613069593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613069603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613069603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613069613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613069613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:49,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/948bed12ae7448deb6d316c2ea5a46a0 2024-11-14T19:36:49,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/d170364908f6429297b35c8e8dc1381b is 50, key is test_row_0/C:col10/1731613008801/Put/seqid=0 2024-11-14T19:36:49,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742235_1411 (size=12001) 2024-11-14T19:36:49,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/d170364908f6429297b35c8e8dc1381b 2024-11-14T19:36:49,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/5f82da6fe0594c239af09d4be14e19c3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/5f82da6fe0594c239af09d4be14e19c3 2024-11-14T19:36:49,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/5f82da6fe0594c239af09d4be14e19c3, entries=250, sequenceid=12, filesize=16.3 K 2024-11-14T19:36:49,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/948bed12ae7448deb6d316c2ea5a46a0 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/948bed12ae7448deb6d316c2ea5a46a0 2024-11-14T19:36:49,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:49,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:49,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/948bed12ae7448deb6d316c2ea5a46a0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T19:36:49,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/d170364908f6429297b35c8e8dc1381b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d170364908f6429297b35c8e8dc1381b 2024-11-14T19:36:49,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d170364908f6429297b35c8e8dc1381b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-14T19:36:49,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for ae3a414a536a107c2204f2532a580772 in 1081ms, sequenceid=12, compaction requested=false 2024-11-14T19:36:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T19:36:49,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T19:36:50,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-14T19:36:50,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:50,041 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-14T19:36:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:50,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:36:50,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8cb200642a4e47b4815ee3bfd4332529 is 50, key is test_row_0/A:col10/1731613008947/Put/seqid=0 2024-11-14T19:36:50,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:50,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742236_1412 (size=12001) 2024-11-14T19:36:50,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613070149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613070151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613070152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613070172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613070174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613070283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613070283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613070283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613070305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613070305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613070503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613070503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613070503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613070513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613070513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,552 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8cb200642a4e47b4815ee3bfd4332529 2024-11-14T19:36:50,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/1eab5268eaac4b52bc52afb3f0262d56 is 50, key is test_row_0/B:col10/1731613008947/Put/seqid=0 2024-11-14T19:36:50,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742237_1413 (size=12001) 2024-11-14T19:36:50,605 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/1eab5268eaac4b52bc52afb3f0262d56 2024-11-14T19:36:50,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/a165f84af7784c36bce212903c18ccc6 is 50, key is test_row_0/C:col10/1731613008947/Put/seqid=0 2024-11-14T19:36:50,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742238_1414 (size=12001) 2024-11-14T19:36:50,686 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/a165f84af7784c36bce212903c18ccc6 2024-11-14T19:36:50,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8cb200642a4e47b4815ee3bfd4332529 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8cb200642a4e47b4815ee3bfd4332529 2024-11-14T19:36:50,696 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8cb200642a4e47b4815ee3bfd4332529, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T19:36:50,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/1eab5268eaac4b52bc52afb3f0262d56 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1eab5268eaac4b52bc52afb3f0262d56 2024-11-14T19:36:50,707 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1eab5268eaac4b52bc52afb3f0262d56, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T19:36:50,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/a165f84af7784c36bce212903c18ccc6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a165f84af7784c36bce212903c18ccc6 2024-11-14T19:36:50,711 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a165f84af7784c36bce212903c18ccc6, entries=150, sequenceid=47, filesize=11.7 K 2024-11-14T19:36:50,712 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for ae3a414a536a107c2204f2532a580772 in 672ms, sequenceid=47, compaction requested=false 2024-11-14T19:36:50,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:50,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:50,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-14T19:36:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-14T19:36:50,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-14T19:36:50,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9060 sec 2024-11-14T19:36:50,718 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.9210 sec 2024-11-14T19:36:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:50,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:36:50,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:50,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:50,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:50,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:50,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:50,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:50,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/a45cb225ba244104b3012a973b519245 is 50, key is test_row_0/A:col10/1731613010148/Put/seqid=0 2024-11-14T19:36:50,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742239_1415 (size=12001) 2024-11-14T19:36:50,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/a45cb225ba244104b3012a973b519245 2024-11-14T19:36:50,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/b47579d4138147ca853651f72f16188d is 50, key is test_row_0/B:col10/1731613010148/Put/seqid=0 2024-11-14T19:36:50,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742240_1416 (size=12001) 2024-11-14T19:36:50,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613070884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/b47579d4138147ca853651f72f16188d 2024-11-14T19:36:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/7e9d136f752e46b9aae05343b75fd893 is 50, key is test_row_0/C:col10/1731613010148/Put/seqid=0 2024-11-14T19:36:50,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613070895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613070895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613070898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613070899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:50,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-14T19:36:50,945 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:50,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-14T19:36:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T19:36:50,948 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:50,949 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:50,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742241_1417 (size=12001) 2024-11-14T19:36:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613071003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613071023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613071023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613071033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613071033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T19:36:51,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T19:36:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:51,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:51,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613071233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T19:36:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:51,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:51,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613071254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613071254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613071254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613071254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T19:36:51,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/7e9d136f752e46b9aae05343b75fd893 2024-11-14T19:36:51,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/a45cb225ba244104b3012a973b519245 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a45cb225ba244104b3012a973b519245 2024-11-14T19:36:51,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a45cb225ba244104b3012a973b519245, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T19:36:51,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/b47579d4138147ca853651f72f16188d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b47579d4138147ca853651f72f16188d 2024-11-14T19:36:51,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b47579d4138147ca853651f72f16188d, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T19:36:51,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/7e9d136f752e46b9aae05343b75fd893 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/7e9d136f752e46b9aae05343b75fd893 2024-11-14T19:36:51,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/7e9d136f752e46b9aae05343b75fd893, entries=150, sequenceid=59, filesize=11.7 K 2024-11-14T19:36:51,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae3a414a536a107c2204f2532a580772 in 585ms, sequenceid=59, compaction requested=true 2024-11-14T19:36:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:51,406 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:51,406 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:51,407 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:51,407 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:51,407 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:36:51,407 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:36:51,408 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,408 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,408 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/948bed12ae7448deb6d316c2ea5a46a0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1eab5268eaac4b52bc52afb3f0262d56, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b47579d4138147ca853651f72f16188d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.2 K 2024-11-14T19:36:51,408 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/5f82da6fe0594c239af09d4be14e19c3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8cb200642a4e47b4815ee3bfd4332529, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a45cb225ba244104b3012a973b519245] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=39.7 K 2024-11-14T19:36:51,408 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f82da6fe0594c239af09d4be14e19c3, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731613008786 2024-11-14T19:36:51,408 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948bed12ae7448deb6d316c2ea5a46a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731613008786 2024-11-14T19:36:51,409 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eab5268eaac4b52bc52afb3f0262d56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731613008931 2024-11-14T19:36:51,409 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cb200642a4e47b4815ee3bfd4332529, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731613008931 2024-11-14T19:36:51,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-14T19:36:51,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,413 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:36:51,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:51,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:51,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:51,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:51,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:51,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:51,415 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b47579d4138147ca853651f72f16188d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731613010148 2024-11-14T19:36:51,415 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a45cb225ba244104b3012a973b519245, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731613010148 2024-11-14T19:36:51,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/a3e8321017d744b4a500c4c6b638bf8a is 50, key is test_row_0/A:col10/1731613010897/Put/seqid=0 2024-11-14T19:36:51,430 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:51,430 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:51,430 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/b61dc17042cd4cbfbd1d50d252399580 is 50, key is test_row_0/A:col10/1731613010148/Put/seqid=0 2024-11-14T19:36:51,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/5598e75bfb004163bbb04f396077f465 is 50, key is test_row_0/B:col10/1731613010148/Put/seqid=0 2024-11-14T19:36:51,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742242_1418 (size=12104) 2024-11-14T19:36:51,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742243_1419 (size=12001) 2024-11-14T19:36:51,527 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/a3e8321017d744b4a500c4c6b638bf8a 2024-11-14T19:36:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/17ab52a43c6f4c81a6824e7aa276914f is 50, key is test_row_0/B:col10/1731613010897/Put/seqid=0 2024-11-14T19:36:51,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742244_1420 (size=12104) 2024-11-14T19:36:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:51,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:51,574 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/5598e75bfb004163bbb04f396077f465 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5598e75bfb004163bbb04f396077f465 2024-11-14T19:36:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T19:36:51,581 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into 5598e75bfb004163bbb04f396077f465(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:51,581 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:51,581 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=13, startTime=1731613011406; duration=0sec 2024-11-14T19:36:51,581 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:51,583 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:36:51,583 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:51,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742245_1421 (size=12001) 2024-11-14T19:36:51,592 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:51,592 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:36:51,592 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:51,592 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d170364908f6429297b35c8e8dc1381b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a165f84af7784c36bce212903c18ccc6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/7e9d136f752e46b9aae05343b75fd893] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.2 K 2024-11-14T19:36:51,594 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d170364908f6429297b35c8e8dc1381b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731613008786 2024-11-14T19:36:51,594 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a165f84af7784c36bce212903c18ccc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731613008931 2024-11-14T19:36:51,595 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e9d136f752e46b9aae05343b75fd893, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731613010148 2024-11-14T19:36:51,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613071591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613071593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,617 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,617 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/3d64a382d0f84a5cb31cc90e0bb1844b is 50, key is test_row_0/C:col10/1731613010148/Put/seqid=0 2024-11-14T19:36:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613071593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613071595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613071611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742246_1422 (size=12104) 2024-11-14T19:36:51,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613071723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,732 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613071723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613071724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613071724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613071744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,889 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/b61dc17042cd4cbfbd1d50d252399580 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b61dc17042cd4cbfbd1d50d252399580 2024-11-14T19:36:51,893 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into b61dc17042cd4cbfbd1d50d252399580(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:51,893 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:51,893 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=13, startTime=1731613011406; duration=0sec 2024-11-14T19:36:51,893 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:51,893 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:36:51,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613071943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613071943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613071943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613071943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613071965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:51,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:51,992 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/17ab52a43c6f4c81a6824e7aa276914f 2024-11-14T19:36:52,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/79153cb11e414b5184bd4b663f65b976 is 50, key is test_row_0/C:col10/1731613010897/Put/seqid=0 2024-11-14T19:36:52,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742247_1423 (size=12001) 2024-11-14T19:36:52,078 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/3d64a382d0f84a5cb31cc90e0bb1844b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/3d64a382d0f84a5cb31cc90e0bb1844b 2024-11-14T19:36:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T19:36:52,084 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into 3d64a382d0f84a5cb31cc90e0bb1844b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:52,084 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:52,085 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=13, startTime=1731613011406; duration=0sec 2024-11-14T19:36:52,085 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:52,085 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:36:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613072253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613072253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613072264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613072264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613072283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,432 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/79153cb11e414b5184bd4b663f65b976 2024-11-14T19:36:52,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/a3e8321017d744b4a500c4c6b638bf8a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a3e8321017d744b4a500c4c6b638bf8a 2024-11-14T19:36:52,440 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a3e8321017d744b4a500c4c6b638bf8a, entries=150, sequenceid=79, filesize=11.7 K 2024-11-14T19:36:52,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/17ab52a43c6f4c81a6824e7aa276914f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/17ab52a43c6f4c81a6824e7aa276914f 2024-11-14T19:36:52,445 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/17ab52a43c6f4c81a6824e7aa276914f, entries=150, sequenceid=79, filesize=11.7 K 2024-11-14T19:36:52,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/79153cb11e414b5184bd4b663f65b976 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/79153cb11e414b5184bd4b663f65b976 2024-11-14T19:36:52,451 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/79153cb11e414b5184bd4b663f65b976, entries=150, sequenceid=79, filesize=11.7 K 2024-11-14T19:36:52,452 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ae3a414a536a107c2204f2532a580772 in 1039ms, sequenceid=79, compaction requested=false 2024-11-14T19:36:52,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:52,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:52,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-14T19:36:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-14T19:36:52,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-14T19:36:52,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-11-14T19:36:52,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.5150 sec 2024-11-14T19:36:52,647 WARN [regionserver/867b237d0fa7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-14T19:36:52,648 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/5f82da6fe0594c239af09d4be14e19c3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8cb200642a4e47b4815ee3bfd4332529, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a45cb225ba244104b3012a973b519245] to archive 2024-11-14T19:36:52,649 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:52,651 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/5f82da6fe0594c239af09d4be14e19c3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/5f82da6fe0594c239af09d4be14e19c3 2024-11-14T19:36:52,652 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8cb200642a4e47b4815ee3bfd4332529 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8cb200642a4e47b4815ee3bfd4332529 2024-11-14T19:36:52,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a45cb225ba244104b3012a973b519245 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a45cb225ba244104b3012a973b519245 2024-11-14T19:36:52,655 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/948bed12ae7448deb6d316c2ea5a46a0, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1eab5268eaac4b52bc52afb3f0262d56, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b47579d4138147ca853651f72f16188d] to archive 2024-11-14T19:36:52,657 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:52,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/948bed12ae7448deb6d316c2ea5a46a0 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/948bed12ae7448deb6d316c2ea5a46a0 2024-11-14T19:36:52,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1eab5268eaac4b52bc52afb3f0262d56 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1eab5268eaac4b52bc52afb3f0262d56 2024-11-14T19:36:52,669 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b47579d4138147ca853651f72f16188d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b47579d4138147ca853651f72f16188d 2024-11-14T19:36:52,670 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d170364908f6429297b35c8e8dc1381b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a165f84af7784c36bce212903c18ccc6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/7e9d136f752e46b9aae05343b75fd893] to archive 2024-11-14T19:36:52,672 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:36:52,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d170364908f6429297b35c8e8dc1381b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d170364908f6429297b35c8e8dc1381b 2024-11-14T19:36:52,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a165f84af7784c36bce212903c18ccc6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a165f84af7784c36bce212903c18ccc6 2024-11-14T19:36:52,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/867b237d0fa7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/7e9d136f752e46b9aae05343b75fd893 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/7e9d136f752e46b9aae05343b75fd893 2024-11-14T19:36:52,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:52,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:36:52,780 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T19:36:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:52,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/c26b9929763e423d9623c2b34a6d7617 is 50, key is test_row_0/A:col10/1731613012777/Put/seqid=0 2024-11-14T19:36:52,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742248_1424 (size=12001) 2024-11-14T19:36:52,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/c26b9929763e423d9623c2b34a6d7617 2024-11-14T19:36:52,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/5ee11a6ca85b4b158418b51a1e97ab6f is 50, key is test_row_0/B:col10/1731613012777/Put/seqid=0 2024-11-14T19:36:52,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613072828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613072830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613072831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613072832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613072835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742249_1425 (size=12001) 2024-11-14T19:36:52,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/5ee11a6ca85b4b158418b51a1e97ab6f 2024-11-14T19:36:52,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/9fdeb7c2554e46be95eb8455d7b8ab7c is 50, key is test_row_0/C:col10/1731613012777/Put/seqid=0 2024-11-14T19:36:52,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742250_1426 (size=12001) 2024-11-14T19:36:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613072953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613072957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613072963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613072963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613072963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-14T19:36:53,094 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:53,094 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-14T19:36:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T19:36:53,096 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:53,097 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:53,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:53,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613073173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613073184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613073184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613073184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613073185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T19:36:53,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T19:36:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:53,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:53,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:53,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:53,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/9fdeb7c2554e46be95eb8455d7b8ab7c 2024-11-14T19:36:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/c26b9929763e423d9623c2b34a6d7617 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c26b9929763e423d9623c2b34a6d7617 2024-11-14T19:36:53,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c26b9929763e423d9623c2b34a6d7617, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T19:36:53,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/5ee11a6ca85b4b158418b51a1e97ab6f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5ee11a6ca85b4b158418b51a1e97ab6f 2024-11-14T19:36:53,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5ee11a6ca85b4b158418b51a1e97ab6f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T19:36:53,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/9fdeb7c2554e46be95eb8455d7b8ab7c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/9fdeb7c2554e46be95eb8455d7b8ab7c 2024-11-14T19:36:53,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/9fdeb7c2554e46be95eb8455d7b8ab7c, entries=150, sequenceid=96, filesize=11.7 K 2024-11-14T19:36:53,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ae3a414a536a107c2204f2532a580772 in 579ms, sequenceid=96, compaction requested=true 2024-11-14T19:36:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:53,357 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:53,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:53,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:53,358 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:53,359 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:53,359 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:53,359 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:36:53,359 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:53,359 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5598e75bfb004163bbb04f396077f465, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/17ab52a43c6f4c81a6824e7aa276914f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5ee11a6ca85b4b158418b51a1e97ab6f] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.3 K 2024-11-14T19:36:53,359 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:36:53,359 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:53,359 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b61dc17042cd4cbfbd1d50d252399580, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a3e8321017d744b4a500c4c6b638bf8a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c26b9929763e423d9623c2b34a6d7617] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.3 K 2024-11-14T19:36:53,359 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 5598e75bfb004163bbb04f396077f465, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731613010148 2024-11-14T19:36:53,360 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b61dc17042cd4cbfbd1d50d252399580, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731613010148 2024-11-14T19:36:53,360 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3e8321017d744b4a500c4c6b638bf8a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731613010891 2024-11-14T19:36:53,360 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ab52a43c6f4c81a6824e7aa276914f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731613010891 2024-11-14T19:36:53,360 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c26b9929763e423d9623c2b34a6d7617, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731613011609 2024-11-14T19:36:53,360 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ee11a6ca85b4b158418b51a1e97ab6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731613011609 2024-11-14T19:36:53,372 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:53,373 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/d11975e24a7646cf8c7f9e747a023e87 is 50, key is test_row_0/B:col10/1731613012777/Put/seqid=0 2024-11-14T19:36:53,385 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:53,385 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/24ee807c9c0e467ab778a2d0b5ee9088 is 50, key is test_row_0/A:col10/1731613012777/Put/seqid=0 2024-11-14T19:36:53,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-14T19:36:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:53,402 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:53,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:53,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742252_1428 (size=12104) 2024-11-14T19:36:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:53,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T19:36:53,417 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/24ee807c9c0e467ab778a2d0b5ee9088 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/24ee807c9c0e467ab778a2d0b5ee9088 2024-11-14T19:36:53,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742251_1427 (size=12104) 2024-11-14T19:36:53,430 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into 24ee807c9c0e467ab778a2d0b5ee9088(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:53,430 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:53,430 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=13, startTime=1731613013357; duration=0sec 2024-11-14T19:36:53,430 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:53,430 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:36:53,430 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:53,433 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:53,433 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:36:53,434 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:53,434 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/3d64a382d0f84a5cb31cc90e0bb1844b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/79153cb11e414b5184bd4b663f65b976, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/9fdeb7c2554e46be95eb8455d7b8ab7c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.3 K 2024-11-14T19:36:53,434 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d64a382d0f84a5cb31cc90e0bb1844b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731613010148 2024-11-14T19:36:53,439 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79153cb11e414b5184bd4b663f65b976, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731613010891 2024-11-14T19:36:53,440 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/d11975e24a7646cf8c7f9e747a023e87 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/d11975e24a7646cf8c7f9e747a023e87 2024-11-14T19:36:53,440 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fdeb7c2554e46be95eb8455d7b8ab7c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731613011609 2024-11-14T19:36:53,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/36daa09f1b14484cabb763835a5419c7 is 50, key is test_row_0/A:col10/1731613012823/Put/seqid=0 2024-11-14T19:36:53,451 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into d11975e24a7646cf8c7f9e747a023e87(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:53,452 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:53,452 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=13, startTime=1731613013357; duration=0sec 2024-11-14T19:36:53,452 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:53,452 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:36:53,472 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:53,472 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/a396fe54f28c42f6ab1ae72bd0ee5a29 is 50, key is test_row_0/C:col10/1731613012777/Put/seqid=0 2024-11-14T19:36:53,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742253_1429 (size=12001) 2024-11-14T19:36:53,493 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/36daa09f1b14484cabb763835a5419c7 2024-11-14T19:36:53,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3a4ad9167a944c6591e3a0b32c8ca099 is 50, key is test_row_0/B:col10/1731613012823/Put/seqid=0 2024-11-14T19:36:53,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613073519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742254_1430 (size=12104) 2024-11-14T19:36:53,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613073521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613073522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613073522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,537 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613073523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,548 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/a396fe54f28c42f6ab1ae72bd0ee5a29 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a396fe54f28c42f6ab1ae72bd0ee5a29 2024-11-14T19:36:53,558 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into a396fe54f28c42f6ab1ae72bd0ee5a29(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:53,558 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:53,558 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=13, startTime=1731613013357; duration=0sec 2024-11-14T19:36:53,558 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:53,558 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:36:53,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742255_1431 (size=12001) 2024-11-14T19:36:53,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613073633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613073644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613073644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613073645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613073645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T19:36:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613073845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613073853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613073863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613073863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613073864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:53,961 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3a4ad9167a944c6591e3a0b32c8ca099 2024-11-14T19:36:53,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/501489ca0fcd43ffa99735dd771a4eee is 50, key is test_row_0/C:col10/1731613012823/Put/seqid=0 2024-11-14T19:36:53,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742256_1432 (size=12001) 2024-11-14T19:36:54,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613074163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613074163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613074183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613074183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613074183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T19:36:54,399 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/501489ca0fcd43ffa99735dd771a4eee 2024-11-14T19:36:54,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/36daa09f1b14484cabb763835a5419c7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/36daa09f1b14484cabb763835a5419c7 2024-11-14T19:36:54,410 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/36daa09f1b14484cabb763835a5419c7, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T19:36:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3a4ad9167a944c6591e3a0b32c8ca099 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3a4ad9167a944c6591e3a0b32c8ca099 2024-11-14T19:36:54,415 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3a4ad9167a944c6591e3a0b32c8ca099, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T19:36:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/501489ca0fcd43ffa99735dd771a4eee as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/501489ca0fcd43ffa99735dd771a4eee 2024-11-14T19:36:54,426 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/501489ca0fcd43ffa99735dd771a4eee, entries=150, sequenceid=114, filesize=11.7 K 2024-11-14T19:36:54,427 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for ae3a414a536a107c2204f2532a580772 in 1025ms, sequenceid=114, compaction requested=false 2024-11-14T19:36:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-14T19:36:54,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-14T19:36:54,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-14T19:36:54,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3320 sec 2024-11-14T19:36:54,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.3410 sec 2024-11-14T19:36:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:54,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T19:36:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:54,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/1778555eeb144e4597777ea4d42d8d4e is 50, key is test_row_0/A:col10/1731613014680/Put/seqid=0 2024-11-14T19:36:54,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742257_1433 (size=14491) 2024-11-14T19:36:54,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/1778555eeb144e4597777ea4d42d8d4e 2024-11-14T19:36:54,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/81e06dbe576746abb783b9485bf996bc is 50, key is test_row_0/B:col10/1731613014680/Put/seqid=0 2024-11-14T19:36:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613074720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613074721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613074734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742258_1434 (size=12151) 2024-11-14T19:36:54,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613074742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613074745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,761 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613074853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613074853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613074853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613074863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:54,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613074873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613075073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613075073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613075073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613075083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613075093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/81e06dbe576746abb783b9485bf996bc 2024-11-14T19:36:55,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/18cfa11c6dcd4142b2453259ca3f5bb6 is 50, key is test_row_0/C:col10/1731613014680/Put/seqid=0 2024-11-14T19:36:55,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742259_1435 (size=12151) 2024-11-14T19:36:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-14T19:36:55,243 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:55,244 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:55,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/18cfa11c6dcd4142b2453259ca3f5bb6 2024-11-14T19:36:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-14T19:36:55,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:55,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T19:36:55,247 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:55,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:55,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/1778555eeb144e4597777ea4d42d8d4e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/1778555eeb144e4597777ea4d42d8d4e 2024-11-14T19:36:55,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/1778555eeb144e4597777ea4d42d8d4e, entries=200, sequenceid=134, filesize=14.2 K 2024-11-14T19:36:55,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/81e06dbe576746abb783b9485bf996bc as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/81e06dbe576746abb783b9485bf996bc 2024-11-14T19:36:55,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/81e06dbe576746abb783b9485bf996bc, entries=150, sequenceid=134, filesize=11.9 K 2024-11-14T19:36:55,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/18cfa11c6dcd4142b2453259ca3f5bb6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18cfa11c6dcd4142b2453259ca3f5bb6 2024-11-14T19:36:55,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18cfa11c6dcd4142b2453259ca3f5bb6, entries=150, sequenceid=134, filesize=11.9 K 2024-11-14T19:36:55,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ae3a414a536a107c2204f2532a580772 in 596ms, sequenceid=134, compaction requested=true 2024-11-14T19:36:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:55,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:55,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:55,277 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:55,277 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:55,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:55,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:55,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:55,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:55,278 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:55,278 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:36:55,278 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,278 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/d11975e24a7646cf8c7f9e747a023e87, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3a4ad9167a944c6591e3a0b32c8ca099, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/81e06dbe576746abb783b9485bf996bc] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.4 K 2024-11-14T19:36:55,278 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38596 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:55,278 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:36:55,279 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,279 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/24ee807c9c0e467ab778a2d0b5ee9088, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/36daa09f1b14484cabb763835a5419c7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/1778555eeb144e4597777ea4d42d8d4e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=37.7 K 2024-11-14T19:36:55,279 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 24ee807c9c0e467ab778a2d0b5ee9088, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731613011609 2024-11-14T19:36:55,279 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d11975e24a7646cf8c7f9e747a023e87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731613011609 2024-11-14T19:36:55,280 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 36daa09f1b14484cabb763835a5419c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731613012823 2024-11-14T19:36:55,280 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a4ad9167a944c6591e3a0b32c8ca099, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731613012823 2024-11-14T19:36:55,280 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1778555eeb144e4597777ea4d42d8d4e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731613013517 2024-11-14T19:36:55,280 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e06dbe576746abb783b9485bf996bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731613013520 2024-11-14T19:36:55,291 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:55,292 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/c7fb3058806a44acb21212ee089d69fe is 50, key is test_row_0/A:col10/1731613014680/Put/seqid=0 2024-11-14T19:36:55,303 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:55,303 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/7573533b1fc74f938dc8472a3890480d is 50, key is test_row_0/B:col10/1731613014680/Put/seqid=0 2024-11-14T19:36:55,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742260_1436 (size=12357) 2024-11-14T19:36:55,345 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/c7fb3058806a44acb21212ee089d69fe as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c7fb3058806a44acb21212ee089d69fe 2024-11-14T19:36:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T19:36:55,357 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into c7fb3058806a44acb21212ee089d69fe(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:55,357 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:55,357 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=13, startTime=1731613015276; duration=0sec 2024-11-14T19:36:55,357 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:55,357 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:36:55,358 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:55,358 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:55,359 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:36:55,359 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,359 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a396fe54f28c42f6ab1ae72bd0ee5a29, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/501489ca0fcd43ffa99735dd771a4eee, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18cfa11c6dcd4142b2453259ca3f5bb6] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.4 K 2024-11-14T19:36:55,359 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a396fe54f28c42f6ab1ae72bd0ee5a29, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731613011609 2024-11-14T19:36:55,359 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 501489ca0fcd43ffa99735dd771a4eee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731613012823 2024-11-14T19:36:55,360 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 18cfa11c6dcd4142b2453259ca3f5bb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731613013520 2024-11-14T19:36:55,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742261_1437 (size=12357) 2024-11-14T19:36:55,368 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/7573533b1fc74f938dc8472a3890480d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/7573533b1fc74f938dc8472a3890480d 2024-11-14T19:36:55,380 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:55,380 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/267d0e4cb39044c48aad979ba2c857f1 is 50, key is test_row_0/C:col10/1731613014680/Put/seqid=0 2024-11-14T19:36:55,382 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into 7573533b1fc74f938dc8472a3890480d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:55,382 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:55,382 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=13, startTime=1731613015277; duration=0sec 2024-11-14T19:36:55,382 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:55,382 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:36:55,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742262_1438 (size=12357) 2024-11-14T19:36:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:55,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T19:36:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:55,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T19:36:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/86b30276f11745008ee37f41bdc5bd5e is 50, key is test_row_0/A:col10/1731613015396/Put/seqid=0 2024-11-14T19:36:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,410 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/267d0e4cb39044c48aad979ba2c857f1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/267d0e4cb39044c48aad979ba2c857f1 2024-11-14T19:36:55,425 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into 267d0e4cb39044c48aad979ba2c857f1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:55,425 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:55,425 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=13, startTime=1731613015278; duration=0sec 2024-11-14T19:36:55,425 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:55,425 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:36:55,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742263_1439 (size=14541) 2024-11-14T19:36:55,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613075434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613075437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613075439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613075455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613075456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T19:36:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T19:36:55,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613075564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613075565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613075565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613075573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613075573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T19:36:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613075773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613075783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613075784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613075794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613075794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:55,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:55,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/86b30276f11745008ee37f41bdc5bd5e 2024-11-14T19:36:55,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T19:36:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:55,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/cd98c8b2b0764b7d9bb77a01ae045dc8 is 50, key is test_row_0/B:col10/1731613015396/Put/seqid=0 2024-11-14T19:36:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T19:36:55,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742264_1440 (size=12151) 2024-11-14T19:36:55,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/cd98c8b2b0764b7d9bb77a01ae045dc8 2024-11-14T19:36:55,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/46c09534787c4b1ea629f2c8f21910b3 is 50, key is test_row_0/C:col10/1731613015396/Put/seqid=0 2024-11-14T19:36:55,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742265_1441 (size=12151) 2024-11-14T19:36:55,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/46c09534787c4b1ea629f2c8f21910b3 2024-11-14T19:36:55,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/86b30276f11745008ee37f41bdc5bd5e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/86b30276f11745008ee37f41bdc5bd5e 2024-11-14T19:36:55,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/86b30276f11745008ee37f41bdc5bd5e, entries=200, sequenceid=152, filesize=14.2 K 2024-11-14T19:36:55,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/cd98c8b2b0764b7d9bb77a01ae045dc8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/cd98c8b2b0764b7d9bb77a01ae045dc8 2024-11-14T19:36:55,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/cd98c8b2b0764b7d9bb77a01ae045dc8, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T19:36:55,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/46c09534787c4b1ea629f2c8f21910b3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/46c09534787c4b1ea629f2c8f21910b3 2024-11-14T19:36:55,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/46c09534787c4b1ea629f2c8f21910b3, entries=150, sequenceid=152, filesize=11.9 K 2024-11-14T19:36:55,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ae3a414a536a107c2204f2532a580772 in 583ms, sequenceid=152, compaction requested=false 2024-11-14T19:36:55,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:56,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-14T19:36:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,023 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-14T19:36:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8ea04a3fcc4c419284d65e1e533d2a0e is 50, key is test_row_0/A:col10/1731613015454/Put/seqid=0 2024-11-14T19:36:56,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742266_1442 (size=12151) 2024-11-14T19:36:56,068 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8ea04a3fcc4c419284d65e1e533d2a0e 2024-11-14T19:36:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3569e1f2034844b8bc78185592ff0bd9 is 50, key is test_row_0/B:col10/1731613015454/Put/seqid=0 2024-11-14T19:36:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:56,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:56,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742267_1443 (size=12151) 2024-11-14T19:36:56,112 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3569e1f2034844b8bc78185592ff0bd9 2024-11-14T19:36:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/6bbd481eb7bc4c64b3eff5e5a60d5d90 is 50, key is test_row_0/C:col10/1731613015454/Put/seqid=0 2024-11-14T19:36:56,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613076126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613076124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613076134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613076142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613076142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742268_1444 (size=12151) 2024-11-14T19:36:56,168 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/6bbd481eb7bc4c64b3eff5e5a60d5d90 2024-11-14T19:36:56,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8ea04a3fcc4c419284d65e1e533d2a0e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ea04a3fcc4c419284d65e1e533d2a0e 2024-11-14T19:36:56,175 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ea04a3fcc4c419284d65e1e533d2a0e, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T19:36:56,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3569e1f2034844b8bc78185592ff0bd9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3569e1f2034844b8bc78185592ff0bd9 2024-11-14T19:36:56,180 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3569e1f2034844b8bc78185592ff0bd9, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T19:36:56,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/6bbd481eb7bc4c64b3eff5e5a60d5d90 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6bbd481eb7bc4c64b3eff5e5a60d5d90 2024-11-14T19:36:56,185 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6bbd481eb7bc4c64b3eff5e5a60d5d90, entries=150, sequenceid=169, filesize=11.9 K 2024-11-14T19:36:56,186 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ae3a414a536a107c2204f2532a580772 in 163ms, sequenceid=169, compaction requested=true 2024-11-14T19:36:56,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:56,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-14T19:36:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-14T19:36:56,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-14T19:36:56,190 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 941 msec 2024-11-14T19:36:56,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 950 msec 2024-11-14T19:36:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:56,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-14T19:36:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:56,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/f97dd49161ea4038a23997bc2d4dcf7c is 50, key is test_row_0/A:col10/1731613016138/Put/seqid=0 2024-11-14T19:36:56,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742269_1445 (size=14541) 2024-11-14T19:36:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613076296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613076301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613076301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613076302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613076304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-14T19:36:56,384 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:56,384 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:56,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-14T19:36:56,386 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T19:36:56,387 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:56,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613076423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613076423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613076423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613076424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613076433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T19:36:56,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T19:36:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:56,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:56,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613076633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613076634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613076634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613076634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613076653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:56,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:56,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T19:36:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:56,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:56,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:56,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/f97dd49161ea4038a23997bc2d4dcf7c 2024-11-14T19:36:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T19:36:56,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/452f534a24244ebb9f5fa6190abea761 is 50, key is test_row_0/B:col10/1731613016138/Put/seqid=0 2024-11-14T19:36:56,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742270_1446 (size=12151) 2024-11-14T19:36:56,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/452f534a24244ebb9f5fa6190abea761 2024-11-14T19:36:56,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/c4af45ece76f4a69a5f0f7ab52d65b74 is 50, key is test_row_0/C:col10/1731613016138/Put/seqid=0 2024-11-14T19:36:56,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742271_1447 (size=12151) 2024-11-14T19:36:56,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/c4af45ece76f4a69a5f0f7ab52d65b74 2024-11-14T19:36:56,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/f97dd49161ea4038a23997bc2d4dcf7c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/f97dd49161ea4038a23997bc2d4dcf7c 2024-11-14T19:36:56,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/f97dd49161ea4038a23997bc2d4dcf7c, entries=200, sequenceid=186, filesize=14.2 K 2024-11-14T19:36:56,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/452f534a24244ebb9f5fa6190abea761 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/452f534a24244ebb9f5fa6190abea761 2024-11-14T19:36:56,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/452f534a24244ebb9f5fa6190abea761, entries=150, sequenceid=186, filesize=11.9 K 2024-11-14T19:36:56,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/c4af45ece76f4a69a5f0f7ab52d65b74 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c4af45ece76f4a69a5f0f7ab52d65b74 2024-11-14T19:36:56,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c4af45ece76f4a69a5f0f7ab52d65b74, entries=150, sequenceid=186, filesize=11.9 K 2024-11-14T19:36:56,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ae3a414a536a107c2204f2532a580772 in 534ms, sequenceid=186, compaction requested=true 2024-11-14T19:36:56,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:56,796 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:36:56,801 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53590 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:36:56,801 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:36:56,801 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,802 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c7fb3058806a44acb21212ee089d69fe, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/86b30276f11745008ee37f41bdc5bd5e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ea04a3fcc4c419284d65e1e533d2a0e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/f97dd49161ea4038a23997bc2d4dcf7c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=52.3 K 2024-11-14T19:36:56,802 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7fb3058806a44acb21212ee089d69fe, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731613013520 2024-11-14T19:36:56,802 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86b30276f11745008ee37f41bdc5bd5e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731613014716 2024-11-14T19:36:56,803 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ea04a3fcc4c419284d65e1e533d2a0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731613015431 2024-11-14T19:36:56,803 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting f97dd49161ea4038a23997bc2d4dcf7c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731613016122 2024-11-14T19:36:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:56,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:56,808 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:36:56,809 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:36:56,809 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:36:56,809 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,809 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/7573533b1fc74f938dc8472a3890480d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/cd98c8b2b0764b7d9bb77a01ae045dc8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3569e1f2034844b8bc78185592ff0bd9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/452f534a24244ebb9f5fa6190abea761] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=47.7 K 2024-11-14T19:36:56,811 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 7573533b1fc74f938dc8472a3890480d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731613013520 2024-11-14T19:36:56,812 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:56,812 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/eb26f4cc272d40e0a1253e4b3e1c0220 is 50, key is test_row_0/A:col10/1731613016138/Put/seqid=0 2024-11-14T19:36:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:56,814 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cd98c8b2b0764b7d9bb77a01ae045dc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731613014716 2024-11-14T19:36:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:56,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:56,814 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3569e1f2034844b8bc78185592ff0bd9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731613015431 2024-11-14T19:36:56,814 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 452f534a24244ebb9f5fa6190abea761, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731613016138 2024-11-14T19:36:56,837 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:56,837 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/79b191a7216e40caa0b7a2808ae792fb is 50, key is test_row_0/B:col10/1731613016138/Put/seqid=0 2024-11-14T19:36:56,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-14T19:36:56,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,853 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:36:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:56,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:56,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742272_1448 (size=12493) 2024-11-14T19:36:56,885 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/eb26f4cc272d40e0a1253e4b3e1c0220 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/eb26f4cc272d40e0a1253e4b3e1c0220 2024-11-14T19:36:56,892 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into eb26f4cc272d40e0a1253e4b3e1c0220(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:56,892 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:56,892 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=12, startTime=1731613016796; duration=0sec 2024-11-14T19:36:56,892 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:56,892 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:36:56,892 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:36:56,895 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:36:56,895 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:36:56,895 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:56,896 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/267d0e4cb39044c48aad979ba2c857f1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/46c09534787c4b1ea629f2c8f21910b3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6bbd481eb7bc4c64b3eff5e5a60d5d90, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c4af45ece76f4a69a5f0f7ab52d65b74] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=47.7 K 2024-11-14T19:36:56,896 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 267d0e4cb39044c48aad979ba2c857f1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731613013520 2024-11-14T19:36:56,896 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c09534787c4b1ea629f2c8f21910b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731613014716 2024-11-14T19:36:56,896 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bbd481eb7bc4c64b3eff5e5a60d5d90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731613015431 2024-11-14T19:36:56,896 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4af45ece76f4a69a5f0f7ab52d65b74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731613016138 2024-11-14T19:36:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8ae706c3548345fd8a19eead473f29d8 is 50, key is test_row_0/A:col10/1731613016295/Put/seqid=0 2024-11-14T19:36:56,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742273_1449 (size=12493) 2024-11-14T19:36:56,921 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/79b191a7216e40caa0b7a2808ae792fb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/79b191a7216e40caa0b7a2808ae792fb 2024-11-14T19:36:56,933 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into 79b191a7216e40caa0b7a2808ae792fb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:56,933 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:56,933 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=12, startTime=1731613016808; duration=0sec 2024-11-14T19:36:56,933 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:56,933 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:36:56,955 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:56,956 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/90647a67de5d4617b60be161b1a6c473 is 50, key is test_row_0/C:col10/1731613016138/Put/seqid=0 2024-11-14T19:36:56,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:56,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:56,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742274_1450 (size=12151) 2024-11-14T19:36:56,984 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8ae706c3548345fd8a19eead473f29d8 2024-11-14T19:36:56,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742275_1451 (size=12493) 2024-11-14T19:36:57,008 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/90647a67de5d4617b60be161b1a6c473 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/90647a67de5d4617b60be161b1a6c473 2024-11-14T19:36:57,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/15814acbebfe48148c2e73cf22813afb is 50, key is test_row_0/B:col10/1731613016295/Put/seqid=0 2024-11-14T19:36:57,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T19:36:57,015 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into 90647a67de5d4617b60be161b1a6c473(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:57,015 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:57,015 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=12, startTime=1731613016814; duration=0sec 2024-11-14T19:36:57,015 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:57,015 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:36:57,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613076991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613076993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613077006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613077012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613077014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742276_1452 (size=12151) 2024-11-14T19:36:57,060 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/15814acbebfe48148c2e73cf22813afb 2024-11-14T19:36:57,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/179793d8f027458abb44cbee7f5e1c8d is 50, key is test_row_0/C:col10/1731613016295/Put/seqid=0 2024-11-14T19:36:57,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742277_1453 (size=12151) 2024-11-14T19:36:57,105 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/179793d8f027458abb44cbee7f5e1c8d 2024-11-14T19:36:57,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8ae706c3548345fd8a19eead473f29d8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ae706c3548345fd8a19eead473f29d8 2024-11-14T19:36:57,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613077123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613077123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,128 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ae706c3548345fd8a19eead473f29d8, entries=150, sequenceid=201, filesize=11.9 K 2024-11-14T19:36:57,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/15814acbebfe48148c2e73cf22813afb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/15814acbebfe48148c2e73cf22813afb 2024-11-14T19:36:57,136 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/15814acbebfe48148c2e73cf22813afb, entries=150, sequenceid=201, filesize=11.9 K 2024-11-14T19:36:57,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/179793d8f027458abb44cbee7f5e1c8d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/179793d8f027458abb44cbee7f5e1c8d 2024-11-14T19:36:57,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613077135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613077135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,144 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/179793d8f027458abb44cbee7f5e1c8d, entries=150, sequenceid=201, filesize=11.9 K 2024-11-14T19:36:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,149 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ae3a414a536a107c2204f2532a580772 in 296ms, sequenceid=201, compaction requested=false 2024-11-14T19:36:57,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:57,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-14T19:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-14T19:36:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:57,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:57,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:57,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-14T19:36:57,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 763 msec 2024-11-14T19:36:57,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/2f442e3d10c94ea288a2acc20905a324 is 50, key is test_row_0/A:col10/1731613017147/Put/seqid=0 2024-11-14T19:36:57,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 771 msec 2024-11-14T19:36:57,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742278_1454 (size=14541) 2024-11-14T19:36:57,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/2f442e3d10c94ea288a2acc20905a324 2024-11-14T19:36:57,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/31d921ee4a414e26befd064d41dbd9c4 is 50, key is test_row_0/B:col10/1731613017147/Put/seqid=0 2024-11-14T19:36:57,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742279_1455 (size=12151) 2024-11-14T19:36:57,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/31d921ee4a414e26befd064d41dbd9c4 2024-11-14T19:36:57,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/e46a50efe3df47979c77db3050e7f6dd is 50, key is test_row_0/C:col10/1731613017147/Put/seqid=0 2024-11-14T19:36:57,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613077268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742280_1456 (size=12151) 2024-11-14T19:36:57,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/e46a50efe3df47979c77db3050e7f6dd 2024-11-14T19:36:57,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/2f442e3d10c94ea288a2acc20905a324 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2f442e3d10c94ea288a2acc20905a324 2024-11-14T19:36:57,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2f442e3d10c94ea288a2acc20905a324, entries=200, sequenceid=222, filesize=14.2 K 2024-11-14T19:36:57,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/31d921ee4a414e26befd064d41dbd9c4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31d921ee4a414e26befd064d41dbd9c4 2024-11-14T19:36:57,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31d921ee4a414e26befd064d41dbd9c4, entries=150, sequenceid=222, filesize=11.9 K 2024-11-14T19:36:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/e46a50efe3df47979c77db3050e7f6dd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e46a50efe3df47979c77db3050e7f6dd 2024-11-14T19:36:57,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e46a50efe3df47979c77db3050e7f6dd, entries=150, sequenceid=222, filesize=11.9 K 2024-11-14T19:36:57,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ae3a414a536a107c2204f2532a580772 in 189ms, sequenceid=222, compaction requested=true 2024-11-14T19:36:57,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:57,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:36:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:57,339 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:36:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:36:57,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T19:36:57,339 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:57,339 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:57,339 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:36:57,339 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:57,340 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,340 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:36:57,340 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/eb26f4cc272d40e0a1253e4b3e1c0220, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ae706c3548345fd8a19eead473f29d8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2f442e3d10c94ea288a2acc20905a324] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=38.3 K 2024-11-14T19:36:57,340 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,340 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/79b191a7216e40caa0b7a2808ae792fb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/15814acbebfe48148c2e73cf22813afb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31d921ee4a414e26befd064d41dbd9c4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.9 K 2024-11-14T19:36:57,340 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb26f4cc272d40e0a1253e4b3e1c0220, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731613016138 2024-11-14T19:36:57,340 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 79b191a7216e40caa0b7a2808ae792fb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731613016138 2024-11-14T19:36:57,340 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae706c3548345fd8a19eead473f29d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731613016291 2024-11-14T19:36:57,340 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 15814acbebfe48148c2e73cf22813afb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731613016291 2024-11-14T19:36:57,341 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f442e3d10c94ea288a2acc20905a324, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731613016988 2024-11-14T19:36:57,341 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d921ee4a414e26befd064d41dbd9c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731613016988 2024-11-14T19:36:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:57,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:36:57,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:57,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:57,357 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:57,357 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/a8dd56c731bc4165928020965cf6f094 is 50, key is test_row_0/B:col10/1731613017147/Put/seqid=0 2024-11-14T19:36:57,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/02cf717e3b3f4d03bfb9e26e84253935 is 50, key is test_row_0/A:col10/1731613017346/Put/seqid=0 2024-11-14T19:36:57,373 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:57,374 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/0dcd1f74c55f40ceb577decc885c939d is 50, key is test_row_0/A:col10/1731613017147/Put/seqid=0 2024-11-14T19:36:57,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742282_1458 (size=12595) 2024-11-14T19:36:57,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742281_1457 (size=16931) 2024-11-14T19:36:57,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613077397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742283_1459 (size=12595) 2024-11-14T19:36:57,420 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/0dcd1f74c55f40ceb577decc885c939d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0dcd1f74c55f40ceb577decc885c939d 2024-11-14T19:36:57,424 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into 0dcd1f74c55f40ceb577decc885c939d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:57,424 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:57,424 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=13, startTime=1731613017338; duration=0sec 2024-11-14T19:36:57,424 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:36:57,424 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:36:57,424 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:36:57,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613077408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613077409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613077413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613077413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:36:57,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:36:57,431 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,432 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/90647a67de5d4617b60be161b1a6c473, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/179793d8f027458abb44cbee7f5e1c8d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e46a50efe3df47979c77db3050e7f6dd] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=35.9 K 2024-11-14T19:36:57,432 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90647a67de5d4617b60be161b1a6c473, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731613016138 2024-11-14T19:36:57,432 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 179793d8f027458abb44cbee7f5e1c8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731613016291 2024-11-14T19:36:57,432 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e46a50efe3df47979c77db3050e7f6dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731613016988 2024-11-14T19:36:57,461 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#383 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:36:57,461 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/6ab6cd1a2a7c40c586f7d42cd33c595c is 50, key is test_row_0/C:col10/1731613017147/Put/seqid=0 2024-11-14T19:36:57,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742284_1460 (size=12595) 2024-11-14T19:36:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-14T19:36:57,524 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:36:57,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:36:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-14T19:36:57,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T19:36:57,526 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:36:57,527 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:36:57,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:36:57,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613077523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613077533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613077533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613077533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613077533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T19:36:57,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T19:36:57,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:57,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613077743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613077753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613077753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613077753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:57,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613077754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:57,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:57,805 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/a8dd56c731bc4165928020965cf6f094 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/a8dd56c731bc4165928020965cf6f094 2024-11-14T19:36:57,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/02cf717e3b3f4d03bfb9e26e84253935 2024-11-14T19:36:57,817 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into a8dd56c731bc4165928020965cf6f094(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:57,817 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:57,817 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=13, startTime=1731613017339; duration=0sec 2024-11-14T19:36:57,818 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:57,818 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:36:57,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/31093080d4bb41cea7ac6c4e9d351825 is 50, key is test_row_0/B:col10/1731613017346/Put/seqid=0 2024-11-14T19:36:57,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T19:36:57,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:57,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T19:36:57,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742285_1461 (size=12151) 2024-11-14T19:36:57,909 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/6ab6cd1a2a7c40c586f7d42cd33c595c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6ab6cd1a2a7c40c586f7d42cd33c595c 2024-11-14T19:36:57,916 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into 6ab6cd1a2a7c40c586f7d42cd33c595c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:36:57,916 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:57,917 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=13, startTime=1731613017339; duration=0sec 2024-11-14T19:36:57,917 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:36:57,917 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:36:57,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T19:36:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:57,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:57,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613078053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613078080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613078080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613078080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613078080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T19:36:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:58,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:58,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:58,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:58,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T19:36:58,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/31093080d4bb41cea7ac6c4e9d351825 2024-11-14T19:36:58,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/c06ff679ca1e4dc6bd040ba5299a40e9 is 50, key is test_row_0/C:col10/1731613017346/Put/seqid=0 2024-11-14T19:36:58,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T19:36:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:58,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:36:58,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742286_1462 (size=12151) 2024-11-14T19:36:58,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/c06ff679ca1e4dc6bd040ba5299a40e9 2024-11-14T19:36:58,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/02cf717e3b3f4d03bfb9e26e84253935 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/02cf717e3b3f4d03bfb9e26e84253935 2024-11-14T19:36:58,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/02cf717e3b3f4d03bfb9e26e84253935, entries=250, sequenceid=238, filesize=16.5 K 2024-11-14T19:36:58,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/31093080d4bb41cea7ac6c4e9d351825 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31093080d4bb41cea7ac6c4e9d351825 2024-11-14T19:36:58,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31093080d4bb41cea7ac6c4e9d351825, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T19:36:58,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/c06ff679ca1e4dc6bd040ba5299a40e9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c06ff679ca1e4dc6bd040ba5299a40e9 2024-11-14T19:36:58,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c06ff679ca1e4dc6bd040ba5299a40e9, entries=150, sequenceid=238, filesize=11.9 K 2024-11-14T19:36:58,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=100.63 KB/103050 for ae3a414a536a107c2204f2532a580772 in 998ms, sequenceid=238, compaction requested=false 2024-11-14T19:36:58,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:58,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-14T19:36:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:58,449 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-14T19:36:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:36:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:36:58,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:58,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:36:58,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:36:58,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/621844d70d724721ade696808f63a055 is 50, key is test_row_0/A:col10/1731613017408/Put/seqid=0 2024-11-14T19:36:58,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742287_1463 (size=12151) 2024-11-14T19:36:58,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:36:58,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:36:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613078609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613078610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613078609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613078624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613078626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T19:36:58,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613078733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613078733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613078733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613078753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613078754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,901 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/621844d70d724721ade696808f63a055 2024-11-14T19:36:58,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/2c0e6eaea3e0486faa9a2ccc6b656cc4 is 50, key is test_row_0/B:col10/1731613017408/Put/seqid=0 2024-11-14T19:36:58,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613078953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613078953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613078954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742288_1464 (size=12151) 2024-11-14T19:36:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613078973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:58,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613078973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613079264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613079264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613079264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613079293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613079293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,366 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/2c0e6eaea3e0486faa9a2ccc6b656cc4 2024-11-14T19:36:59,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/100646b109dd4eef855a8b84541fda37 is 50, key is test_row_0/C:col10/1731613017408/Put/seqid=0 2024-11-14T19:36:59,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742289_1465 (size=12151) 2024-11-14T19:36:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T19:36:59,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613079773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613079784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613079785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,805 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/100646b109dd4eef855a8b84541fda37 2024-11-14T19:36:59,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/621844d70d724721ade696808f63a055 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/621844d70d724721ade696808f63a055 2024-11-14T19:36:59,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613079803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,815 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/621844d70d724721ade696808f63a055, entries=150, sequenceid=259, filesize=11.9 K 2024-11-14T19:36:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/2c0e6eaea3e0486faa9a2ccc6b656cc4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/2c0e6eaea3e0486faa9a2ccc6b656cc4 2024-11-14T19:36:59,822 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/2c0e6eaea3e0486faa9a2ccc6b656cc4, entries=150, sequenceid=259, filesize=11.9 K 2024-11-14T19:36:59,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/100646b109dd4eef855a8b84541fda37 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/100646b109dd4eef855a8b84541fda37 2024-11-14T19:36:59,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:36:59,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613079813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:36:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:36:59,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:36:59,829 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/100646b109dd4eef855a8b84541fda37, entries=150, sequenceid=259, filesize=11.9 K 2024-11-14T19:36:59,831 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ae3a414a536a107c2204f2532a580772 in 1381ms, sequenceid=259, compaction requested=true 2024-11-14T19:36:59,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:36:59,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:36:59,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-14T19:36:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-14T19:36:59,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-14T19:36:59,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3070 sec 2024-11-14T19:36:59,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.3130 sec 2024-11-14T19:37:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:00,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-14T19:37:00,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:00,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:00,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:00,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:00,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:00,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:00,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/749731a3e3094d0f9f159b5bb1fe0435 is 50, key is test_row_0/A:col10/1731613020799/Put/seqid=0 2024-11-14T19:37:00,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742290_1466 (size=14741) 2024-11-14T19:37:00,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/749731a3e3094d0f9f159b5bb1fe0435 2024-11-14T19:37:00,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/74ad0e77c2054e5aa3eb3ab178df3531 is 50, key is test_row_0/B:col10/1731613020799/Put/seqid=0 2024-11-14T19:37:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613080856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613080856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:00,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613080861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:00,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742291_1467 (size=12301) 2024-11-14T19:37:00,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/74ad0e77c2054e5aa3eb3ab178df3531 2024-11-14T19:37:00,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613080879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613080879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:00,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/942ccceea330467490fdb44e231cca62 is 50, key is test_row_0/C:col10/1731613020799/Put/seqid=0 2024-11-14T19:37:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742292_1468 (size=12301) 2024-11-14T19:37:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613080983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613080984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613080984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:00,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613081003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613081003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613081204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613081204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613081204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613081223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613081223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/942ccceea330467490fdb44e231cca62 2024-11-14T19:37:01,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/749731a3e3094d0f9f159b5bb1fe0435 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/749731a3e3094d0f9f159b5bb1fe0435 2024-11-14T19:37:01,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/749731a3e3094d0f9f159b5bb1fe0435, entries=200, sequenceid=274, filesize=14.4 K 2024-11-14T19:37:01,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/74ad0e77c2054e5aa3eb3ab178df3531 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/74ad0e77c2054e5aa3eb3ab178df3531 2024-11-14T19:37:01,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/74ad0e77c2054e5aa3eb3ab178df3531, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T19:37:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/942ccceea330467490fdb44e231cca62 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/942ccceea330467490fdb44e231cca62 2024-11-14T19:37:01,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/942ccceea330467490fdb44e231cca62, entries=150, sequenceid=274, filesize=12.0 K 2024-11-14T19:37:01,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for ae3a414a536a107c2204f2532a580772 in 563ms, sequenceid=274, compaction requested=true 2024-11-14T19:37:01,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:01,366 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:01,367 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:01,370 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56418 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:01,370 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:37:01,370 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:01,370 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0dcd1f74c55f40ceb577decc885c939d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/02cf717e3b3f4d03bfb9e26e84253935, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/621844d70d724721ade696808f63a055, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/749731a3e3094d0f9f159b5bb1fe0435] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=55.1 K 2024-11-14T19:37:01,370 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:01,370 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:37:01,370 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:01,370 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/a8dd56c731bc4165928020965cf6f094, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31093080d4bb41cea7ac6c4e9d351825, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/2c0e6eaea3e0486faa9a2ccc6b656cc4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/74ad0e77c2054e5aa3eb3ab178df3531] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=48.0 K 2024-11-14T19:37:01,371 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a8dd56c731bc4165928020965cf6f094, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731613016988 2024-11-14T19:37:01,371 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dcd1f74c55f40ceb577decc885c939d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731613016988 2024-11-14T19:37:01,373 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 31093080d4bb41cea7ac6c4e9d351825, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731613017154 2024-11-14T19:37:01,373 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02cf717e3b3f4d03bfb9e26e84253935, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731613017154 2024-11-14T19:37:01,373 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c0e6eaea3e0486faa9a2ccc6b656cc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731613017391 2024-11-14T19:37:01,373 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 621844d70d724721ade696808f63a055, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731613017391 2024-11-14T19:37:01,373 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 74ad0e77c2054e5aa3eb3ab178df3531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731613018607 2024-11-14T19:37:01,374 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749731a3e3094d0f9f159b5bb1fe0435, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731613018607 2024-11-14T19:37:01,385 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:01,385 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:01,386 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/eef26ae77aa9482eafba36dbfbba70e6 is 50, key is test_row_0/B:col10/1731613020799/Put/seqid=0 2024-11-14T19:37:01,386 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8d33ec1cd49b4689b0f0b736563fc95f is 50, key is test_row_0/A:col10/1731613020799/Put/seqid=0 2024-11-14T19:37:01,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742293_1469 (size=12881) 2024-11-14T19:37:01,437 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/8d33ec1cd49b4689b0f0b736563fc95f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8d33ec1cd49b4689b0f0b736563fc95f 2024-11-14T19:37:01,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742294_1470 (size=12881) 2024-11-14T19:37:01,441 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into 8d33ec1cd49b4689b0f0b736563fc95f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:01,441 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:01,441 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=12, startTime=1731613021366; duration=0sec 2024-11-14T19:37:01,441 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:01,441 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:37:01,441 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:01,448 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49198 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:01,448 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:37:01,448 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:01,448 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6ab6cd1a2a7c40c586f7d42cd33c595c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c06ff679ca1e4dc6bd040ba5299a40e9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/100646b109dd4eef855a8b84541fda37, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/942ccceea330467490fdb44e231cca62] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=48.0 K 2024-11-14T19:37:01,448 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab6cd1a2a7c40c586f7d42cd33c595c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731613016988 2024-11-14T19:37:01,449 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c06ff679ca1e4dc6bd040ba5299a40e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731613017154 2024-11-14T19:37:01,451 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100646b109dd4eef855a8b84541fda37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731613017391 2024-11-14T19:37:01,451 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 942ccceea330467490fdb44e231cca62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731613018607 2024-11-14T19:37:01,461 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:01,461 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/386a5d0cbd2545dcb1008c8b70f5cfae is 50, key is test_row_0/C:col10/1731613020799/Put/seqid=0 2024-11-14T19:37:01,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742295_1471 (size=12881) 2024-11-14T19:37:01,494 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/386a5d0cbd2545dcb1008c8b70f5cfae as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/386a5d0cbd2545dcb1008c8b70f5cfae 2024-11-14T19:37:01,501 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into 386a5d0cbd2545dcb1008c8b70f5cfae(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:01,501 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:01,501 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=12, startTime=1731613021367; duration=0sec 2024-11-14T19:37:01,501 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:01,501 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:37:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:01,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731613020850, max timestamp=1731613021525Num uniques 859; 2024-11-14T19:37:01,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:37:01,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731613020850, max timestamp=1731613021525Num uniques 859; 2024-11-14T19:37:01,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:37:01,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=67.09 KB heapSize=184.48 KB 2024-11-14T19:37:01,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:01,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:01,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:01,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:01,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:01,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=179, cellSize=8194, totalHeapSize=18856, min timestamp=1731613020850, max timestamp=1731613021525Num uniques 859; 2024-11-14T19:37:01,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8194, suffix heap size=117064,new segment heap size=18856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=179 2024-11-14T19:37:01,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:01,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/d6fbe60a43a44dbab50d91865bbc889f is 50, key is test_row_0/A:col10/1731613021525/Put/seqid=0 2024-11-14T19:37:01,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742296_1472 (size=12301) 2024-11-14T19:37:01,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/d6fbe60a43a44dbab50d91865bbc889f 2024-11-14T19:37:01,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/1104b10f63d84c5fad0690f56a29111b is 50, key is test_row_0/B:col10/1731613021525/Put/seqid=0 2024-11-14T19:37:01,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742297_1473 (size=12301) 2024-11-14T19:37:01,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/1104b10f63d84c5fad0690f56a29111b 2024-11-14T19:37:01,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/4f85f12f58494883949e8e28464fb9f7 is 50, key is test_row_0/C:col10/1731613021525/Put/seqid=0 2024-11-14T19:37:01,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742298_1474 (size=12301) 2024-11-14T19:37:01,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.84 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/4f85f12f58494883949e8e28464fb9f7 2024-11-14T19:37:01,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/d6fbe60a43a44dbab50d91865bbc889f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/d6fbe60a43a44dbab50d91865bbc889f 2024-11-14T19:37:01,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/d6fbe60a43a44dbab50d91865bbc889f, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T19:37:01,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/1104b10f63d84c5fad0690f56a29111b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1104b10f63d84c5fad0690f56a29111b 2024-11-14T19:37:01,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1104b10f63d84c5fad0690f56a29111b, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T19:37:01,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/4f85f12f58494883949e8e28464fb9f7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/4f85f12f58494883949e8e28464fb9f7 2024-11-14T19:37:01,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/4f85f12f58494883949e8e28464fb9f7, entries=150, sequenceid=297, filesize=12.0 K 2024-11-14T19:37:01,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~36.68 KB/37560, heapSize ~88.53 KB/90656, currentSize=114.05 KB/116790 for ae3a414a536a107c2204f2532a580772 in 105ms, sequenceid=297, compaction requested=false 2024-11-14T19:37:01,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613021587, max timestamp=1731613021635Num uniques 859; 2024-11-14T19:37:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:01,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-14T19:37:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613021587, max timestamp=1731613021626Num uniques 859; 2024-11-14T19:37:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613021587, max timestamp=1731613021626Num uniques 859; 2024-11-14T19:37:01,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:01,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:01,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/634055fda26f4f7ebe88dbd6f25178a4 is 50, key is test_row_0/A:col10/1731613021607/Put/seqid=0 2024-11-14T19:37:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-14T19:37:01,683 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:01,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-14T19:37:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T19:37:01,686 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:01,687 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:01,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:01,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742299_1475 (size=17181) 2024-11-14T19:37:01,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731613021702, max timestamp=1731613021750Num uniques 859; 2024-11-14T19:37:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:37:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613081729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731613021702, max timestamp=1731613021750Num uniques 859; 2024-11-14T19:37:01,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:37:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731613021702, max timestamp=1731613021750Num uniques 859; 2024-11-14T19:37:01,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:37:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T19:37:01,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T19:37:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:01,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:01,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:01,848 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/eef26ae77aa9482eafba36dbfbba70e6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/eef26ae77aa9482eafba36dbfbba70e6 2024-11-14T19:37:01,855 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into eef26ae77aa9482eafba36dbfbba70e6(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:01,855 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:01,855 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=12, startTime=1731613021367; duration=0sec 2024-11-14T19:37:01,855 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:01,855 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:37:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613081826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613081830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613081830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613081849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613081863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613081963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613081963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613081963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613081973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:01,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T19:37:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:01,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:01,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T19:37:02,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/634055fda26f4f7ebe88dbd6f25178a4 2024-11-14T19:37:02,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613082083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3cf8c7e990534affa7a8a8bb76d37db4 is 50, key is test_row_0/B:col10/1731613021607/Put/seqid=0 2024-11-14T19:37:02,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T19:37:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742300_1476 (size=12301) 2024-11-14T19:37:02,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3cf8c7e990534affa7a8a8bb76d37db4 2024-11-14T19:37:02,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613082183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613082183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613082183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/f5273920b58b42138433b62cdfd9640a is 50, key is test_row_0/C:col10/1731613021607/Put/seqid=0 2024-11-14T19:37:02,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613082193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742301_1477 (size=12301) 2024-11-14T19:37:02,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T19:37:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T19:37:02,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613082413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T19:37:02,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:02,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613082503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613082503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613082503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613082513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:02,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:02,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T19:37:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:02,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/f5273920b58b42138433b62cdfd9640a 2024-11-14T19:37:02,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/634055fda26f4f7ebe88dbd6f25178a4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/634055fda26f4f7ebe88dbd6f25178a4 2024-11-14T19:37:02,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/634055fda26f4f7ebe88dbd6f25178a4, entries=250, sequenceid=322, filesize=16.8 K 2024-11-14T19:37:02,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3cf8c7e990534affa7a8a8bb76d37db4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3cf8c7e990534affa7a8a8bb76d37db4 2024-11-14T19:37:02,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3cf8c7e990534affa7a8a8bb76d37db4, entries=150, sequenceid=322, filesize=12.0 K 2024-11-14T19:37:02,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/f5273920b58b42138433b62cdfd9640a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/f5273920b58b42138433b62cdfd9640a 2024-11-14T19:37:02,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/f5273920b58b42138433b62cdfd9640a, entries=150, sequenceid=322, filesize=12.0 K 2024-11-14T19:37:02,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for ae3a414a536a107c2204f2532a580772 in 1019ms, sequenceid=322, compaction requested=true 2024-11-14T19:37:02,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:02,681 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:02,681 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:02,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:02,682 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:02,682 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:02,682 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:37:02,682 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:37:02,682 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,683 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,683 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8d33ec1cd49b4689b0f0b736563fc95f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/d6fbe60a43a44dbab50d91865bbc889f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/634055fda26f4f7ebe88dbd6f25178a4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=41.4 K 2024-11-14T19:37:02,683 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/eef26ae77aa9482eafba36dbfbba70e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1104b10f63d84c5fad0690f56a29111b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3cf8c7e990534affa7a8a8bb76d37db4] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=36.6 K 2024-11-14T19:37:02,683 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting eef26ae77aa9482eafba36dbfbba70e6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731613018607 2024-11-14T19:37:02,683 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d33ec1cd49b4689b0f0b736563fc95f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731613018607 2024-11-14T19:37:02,683 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1104b10f63d84c5fad0690f56a29111b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731613020852 2024-11-14T19:37:02,683 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6fbe60a43a44dbab50d91865bbc889f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731613020852 2024-11-14T19:37:02,686 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cf8c7e990534affa7a8a8bb76d37db4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731613021607 2024-11-14T19:37:02,688 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 634055fda26f4f7ebe88dbd6f25178a4, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731613021607 2024-11-14T19:37:02,708 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#401 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:02,708 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/cf5a4f917f8340bf92eb3062b0657a92 is 50, key is test_row_0/A:col10/1731613021607/Put/seqid=0 2024-11-14T19:37:02,725 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:02,725 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/803f7b7cdae444618554d83bddb6ff8c is 50, key is test_row_0/B:col10/1731613021607/Put/seqid=0 2024-11-14T19:37:02,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-14T19:37:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,763 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-14T19:37:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:02,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:02,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:02,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742302_1478 (size=12983) 2024-11-14T19:37:02,784 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/cf5a4f917f8340bf92eb3062b0657a92 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/cf5a4f917f8340bf92eb3062b0657a92 2024-11-14T19:37:02,791 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into cf5a4f917f8340bf92eb3062b0657a92(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:02,791 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:02,791 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=13, startTime=1731613022681; duration=0sec 2024-11-14T19:37:02,791 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:02,791 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:37:02,791 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:02,792 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:02,792 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:37:02,792 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:02,792 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/386a5d0cbd2545dcb1008c8b70f5cfae, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/4f85f12f58494883949e8e28464fb9f7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/f5273920b58b42138433b62cdfd9640a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=36.6 K 2024-11-14T19:37:02,793 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 386a5d0cbd2545dcb1008c8b70f5cfae, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731613018607 2024-11-14T19:37:02,793 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f85f12f58494883949e8e28464fb9f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731613020852 2024-11-14T19:37:02,794 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5273920b58b42138433b62cdfd9640a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731613021607 2024-11-14T19:37:02,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742303_1479 (size=12983) 2024-11-14T19:37:02,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/ec9dc6b161e748f9b358b30b0ba292c2 is 50, key is test_row_0/A:col10/1731613021797/Put/seqid=0 2024-11-14T19:37:02,816 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/803f7b7cdae444618554d83bddb6ff8c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/803f7b7cdae444618554d83bddb6ff8c 2024-11-14T19:37:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T19:37:02,828 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into 803f7b7cdae444618554d83bddb6ff8c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:02,828 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:02,828 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=13, startTime=1731613022681; duration=0sec 2024-11-14T19:37:02,829 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:02,829 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:37:02,846 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#404 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:02,846 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/e896ba48b5534a06b269910b8b272c2c is 50, key is test_row_0/C:col10/1731613021607/Put/seqid=0 2024-11-14T19:37:02,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742304_1480 (size=12301) 2024-11-14T19:37:02,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742305_1481 (size=12983) 2024-11-14T19:37:02,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:02,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:03,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613083023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613083023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613083023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613083033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613083036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613083153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,248 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/ec9dc6b161e748f9b358b30b0ba292c2 2024-11-14T19:37:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3ae3a77d69b24d0aa16942208b448b41 is 50, key is test_row_0/B:col10/1731613021797/Put/seqid=0 2024-11-14T19:37:03,285 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/e896ba48b5534a06b269910b8b272c2c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e896ba48b5534a06b269910b8b272c2c 2024-11-14T19:37:03,289 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into e896ba48b5534a06b269910b8b272c2c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:03,289 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:03,289 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=13, startTime=1731613022681; duration=0sec 2024-11-14T19:37:03,289 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:03,289 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:37:03,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742306_1482 (size=12301) 2024-11-14T19:37:03,292 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3ae3a77d69b24d0aa16942208b448b41 2024-11-14T19:37:03,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/fca4b3780c30486faa5d221dede682bf is 50, key is test_row_0/C:col10/1731613021797/Put/seqid=0 2024-11-14T19:37:03,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742307_1483 (size=12301) 2024-11-14T19:37:03,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613083373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:03,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613083695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:03,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:03,727 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/fca4b3780c30486faa5d221dede682bf 2024-11-14T19:37:03,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/ec9dc6b161e748f9b358b30b0ba292c2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/ec9dc6b161e748f9b358b30b0ba292c2 2024-11-14T19:37:03,738 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/ec9dc6b161e748f9b358b30b0ba292c2, entries=150, sequenceid=359, filesize=12.0 K 2024-11-14T19:37:03,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3ae3a77d69b24d0aa16942208b448b41 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ae3a77d69b24d0aa16942208b448b41 2024-11-14T19:37:03,747 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ae3a77d69b24d0aa16942208b448b41, entries=150, sequenceid=359, filesize=12.0 K 2024-11-14T19:37:03,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/fca4b3780c30486faa5d221dede682bf as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/fca4b3780c30486faa5d221dede682bf 2024-11-14T19:37:03,756 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/fca4b3780c30486faa5d221dede682bf, entries=150, sequenceid=359, filesize=12.0 K 2024-11-14T19:37:03,757 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ae3a414a536a107c2204f2532a580772 in 994ms, sequenceid=359, compaction requested=false 2024-11-14T19:37:03,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:03,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:03,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-14T19:37:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-14T19:37:03,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-14T19:37:03,773 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0800 sec 2024-11-14T19:37:03,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0900 sec 2024-11-14T19:37:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-14T19:37:03,833 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:03,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-14T19:37:03,838 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:03,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T19:37:03,839 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:03,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T19:37:03,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-14T19:37:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:03,994 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:37:03,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:03,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:03,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:03,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:03,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:03,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:04,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/0e03e01c07644b628ef7693cfd95d018 is 50, key is test_row_1/A:col10/1731613023001/Put/seqid=0 2024-11-14T19:37:04,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742308_1484 (size=9857) 2024-11-14T19:37:04,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:04,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:04,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731613024088, max timestamp=1731613024108Num uniques 859; 2024-11-14T19:37:04,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731613024088, max timestamp=1731613024108Num uniques 859; 2024-11-14T19:37:04,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731613024088, max timestamp=1731613024108Num uniques 859; 2024-11-14T19:37:04,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:37:04,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:37:04,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-14T19:37:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T19:37:04,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613084166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613084167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613084170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613084170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613084213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613084283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613084293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613084293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613084293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,449 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/0e03e01c07644b628ef7693cfd95d018 2024-11-14T19:37:04,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3ba3446a6677431880e7d02d2c9c847e is 50, key is test_row_1/B:col10/1731613023001/Put/seqid=0 2024-11-14T19:37:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T19:37:04,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742309_1485 (size=9857) 2024-11-14T19:37:04,488 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3ba3446a6677431880e7d02d2c9c847e 2024-11-14T19:37:04,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/48abad874da44bd0aeb9395c62581b7c is 50, key is test_row_1/C:col10/1731613023001/Put/seqid=0 2024-11-14T19:37:04,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613084503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613084513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613084514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613084514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742310_1486 (size=9857) 2024-11-14T19:37:04,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613084824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613084833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613084833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:04,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613084833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:04,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:04,938 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/48abad874da44bd0aeb9395c62581b7c 2024-11-14T19:37:04,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/0e03e01c07644b628ef7693cfd95d018 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0e03e01c07644b628ef7693cfd95d018 2024-11-14T19:37:04,950 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0e03e01c07644b628ef7693cfd95d018, entries=100, sequenceid=372, filesize=9.6 K 2024-11-14T19:37:04,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/3ba3446a6677431880e7d02d2c9c847e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ba3446a6677431880e7d02d2c9c847e 2024-11-14T19:37:04,971 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ba3446a6677431880e7d02d2c9c847e, entries=100, sequenceid=372, filesize=9.6 K 2024-11-14T19:37:04,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/48abad874da44bd0aeb9395c62581b7c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/48abad874da44bd0aeb9395c62581b7c 2024-11-14T19:37:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T19:37:04,977 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/48abad874da44bd0aeb9395c62581b7c, entries=100, sequenceid=372, filesize=9.6 K 2024-11-14T19:37:04,984 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ae3a414a536a107c2204f2532a580772 in 990ms, sequenceid=372, compaction requested=true 2024-11-14T19:37:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:04,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-14T19:37:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-14T19:37:04,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-14T19:37:04,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1480 sec 2024-11-14T19:37:04,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.1580 sec 2024-11-14T19:37:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:05,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-14T19:37:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:05,242 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:05,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/375127486cb64bf98ed24d463ece4392 is 50, key is test_row_0/A:col10/1731613024148/Put/seqid=0 2024-11-14T19:37:05,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742311_1487 (size=14741) 2024-11-14T19:37:05,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613085305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613085343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613085353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613085353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613085353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613085422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613085633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/375127486cb64bf98ed24d463ece4392 2024-11-14T19:37:05,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/91a8484aca654cb89588cf7423dcba5a is 50, key is test_row_0/B:col10/1731613024148/Put/seqid=0 2024-11-14T19:37:05,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742312_1488 (size=12301) 2024-11-14T19:37:05,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613085944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:05,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-14T19:37:05,983 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:05,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-14T19:37:05,986 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T19:37:05,987 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:05,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:06,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/91a8484aca654cb89588cf7423dcba5a 2024-11-14T19:37:06,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/38f2992e7cff41a2922307e9a6f4b8d5 is 50, key is test_row_0/C:col10/1731613024148/Put/seqid=0 2024-11-14T19:37:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T19:37:06,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742313_1489 (size=12301) 2024-11-14T19:37:06,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/38f2992e7cff41a2922307e9a6f4b8d5 2024-11-14T19:37:06,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/375127486cb64bf98ed24d463ece4392 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/375127486cb64bf98ed24d463ece4392 2024-11-14T19:37:06,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/375127486cb64bf98ed24d463ece4392, entries=200, sequenceid=409, filesize=14.4 K 2024-11-14T19:37:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/91a8484aca654cb89588cf7423dcba5a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/91a8484aca654cb89588cf7423dcba5a 2024-11-14T19:37:06,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/91a8484aca654cb89588cf7423dcba5a, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T19:37:06,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/38f2992e7cff41a2922307e9a6f4b8d5 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/38f2992e7cff41a2922307e9a6f4b8d5 2024-11-14T19:37:06,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/38f2992e7cff41a2922307e9a6f4b8d5, entries=150, sequenceid=409, filesize=12.0 K 2024-11-14T19:37:06,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for ae3a414a536a107c2204f2532a580772 in 896ms, sequenceid=409, compaction requested=true 2024-11-14T19:37:06,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:06,139 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:06,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-14T19:37:06,139 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:06,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-14T19:37:06,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:06,141 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T19:37:06,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:06,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:06,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:06,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:06,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:06,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:06,145 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:06,145 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:37:06,145 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:06,145 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/803f7b7cdae444618554d83bddb6ff8c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ae3a77d69b24d0aa16942208b448b41, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ba3446a6677431880e7d02d2c9c847e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/91a8484aca654cb89588cf7423dcba5a] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=46.3 K 2024-11-14T19:37:06,145 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49882 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:06,145 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:37:06,145 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:06,146 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/cf5a4f917f8340bf92eb3062b0657a92, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/ec9dc6b161e748f9b358b30b0ba292c2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0e03e01c07644b628ef7693cfd95d018, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/375127486cb64bf98ed24d463ece4392] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=48.7 K 2024-11-14T19:37:06,146 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803f7b7cdae444618554d83bddb6ff8c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731613021607 2024-11-14T19:37:06,146 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cf5a4f917f8340bf92eb3062b0657a92, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731613021607 2024-11-14T19:37:06,147 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9dc6b161e748f9b358b30b0ba292c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731613021797 2024-11-14T19:37:06,147 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae3a77d69b24d0aa16942208b448b41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731613021797 2024-11-14T19:37:06,147 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e03e01c07644b628ef7693cfd95d018, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731613023001 2024-11-14T19:37:06,147 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba3446a6677431880e7d02d2c9c847e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731613023001 2024-11-14T19:37:06,148 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 375127486cb64bf98ed24d463ece4392, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731613024148 2024-11-14T19:37:06,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/7a50eb573a1b4a35ba977eb12a5d7928 is 50, key is test_row_0/A:col10/1731613025299/Put/seqid=0 2024-11-14T19:37:06,173 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a8484aca654cb89588cf7423dcba5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731613024148 2024-11-14T19:37:06,195 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#414 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:06,196 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/0782644511134e0b9a701eda95ca2a31 is 50, key is test_row_0/B:col10/1731613024148/Put/seqid=0 2024-11-14T19:37:06,206 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:06,206 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/2db4958b1bb74125932c629e4d8647e3 is 50, key is test_row_0/A:col10/1731613024148/Put/seqid=0 2024-11-14T19:37:06,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742315_1491 (size=13119) 2024-11-14T19:37:06,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742314_1490 (size=9857) 2024-11-14T19:37:06,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742316_1492 (size=13119) 2024-11-14T19:37:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T19:37:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:06,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:06,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731613026408, max timestamp=1731613026421Num uniques 859; 2024-11-14T19:37:06,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:37:06,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731613026408, max timestamp=1731613026421Num uniques 859; 2024-11-14T19:37:06,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:37:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731613026408, max timestamp=1731613026421Num uniques 859; 2024-11-14T19:37:06,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:37:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731613026448, max timestamp=1731613026463Num uniques 1068; 2024-11-14T19:37:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T19:37:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731613026448, max timestamp=1731613026463Num uniques 1068; 2024-11-14T19:37:06,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T19:37:06,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731613026448, max timestamp=1731613026463Num uniques 1068; 2024-11-14T19:37:06,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-11-14T19:37:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613086497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613086495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613086500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613086515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613086515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T19:37:06,628 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/0782644511134e0b9a701eda95ca2a31 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/0782644511134e0b9a701eda95ca2a31 2024-11-14T19:37:06,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613086623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613086623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,631 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/7a50eb573a1b4a35ba977eb12a5d7928 2024-11-14T19:37:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613086623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613086623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613086623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,633 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into 0782644511134e0b9a701eda95ca2a31(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:06,633 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:06,633 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=12, startTime=1731613026139; duration=0sec 2024-11-14T19:37:06,633 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:06,633 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:37:06,633 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:06,635 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:06,635 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:37:06,635 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:06,635 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e896ba48b5534a06b269910b8b272c2c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/fca4b3780c30486faa5d221dede682bf, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/48abad874da44bd0aeb9395c62581b7c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/38f2992e7cff41a2922307e9a6f4b8d5] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=46.3 K 2024-11-14T19:37:06,635 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e896ba48b5534a06b269910b8b272c2c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731613021607 2024-11-14T19:37:06,636 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca4b3780c30486faa5d221dede682bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731613021797 2024-11-14T19:37:06,636 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48abad874da44bd0aeb9395c62581b7c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731613023001 2024-11-14T19:37:06,636 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f2992e7cff41a2922307e9a6f4b8d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731613024148 2024-11-14T19:37:06,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ee05eb8686f943f58e78b9a31d474f35 is 50, key is test_row_0/B:col10/1731613025299/Put/seqid=0 2024-11-14T19:37:06,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742317_1493 (size=9857) 2024-11-14T19:37:06,642 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:06,642 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ee05eb8686f943f58e78b9a31d474f35 2024-11-14T19:37:06,642 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/069848fc55e04f01ab8c865dc4082a7e is 50, key is test_row_0/C:col10/1731613024148/Put/seqid=0 2024-11-14T19:37:06,646 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/2db4958b1bb74125932c629e4d8647e3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2db4958b1bb74125932c629e4d8647e3 2024-11-14T19:37:06,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742318_1494 (size=13119) 2024-11-14T19:37:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/8eba0387e8484ff4994dec9aa9824879 is 50, key is test_row_0/C:col10/1731613025299/Put/seqid=0 2024-11-14T19:37:06,652 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into 2db4958b1bb74125932c629e4d8647e3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:06,652 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:06,652 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=12, startTime=1731613026138; duration=0sec 2024-11-14T19:37:06,652 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:06,652 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:37:06,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742319_1495 (size=9857) 2024-11-14T19:37:06,654 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/8eba0387e8484ff4994dec9aa9824879 2024-11-14T19:37:06,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/7a50eb573a1b4a35ba977eb12a5d7928 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/7a50eb573a1b4a35ba977eb12a5d7928 2024-11-14T19:37:06,659 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/7a50eb573a1b4a35ba977eb12a5d7928, entries=100, sequenceid=417, filesize=9.6 K 2024-11-14T19:37:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ee05eb8686f943f58e78b9a31d474f35 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ee05eb8686f943f58e78b9a31d474f35 2024-11-14T19:37:06,662 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ee05eb8686f943f58e78b9a31d474f35, entries=100, sequenceid=417, filesize=9.6 K 2024-11-14T19:37:06,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/8eba0387e8484ff4994dec9aa9824879 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/8eba0387e8484ff4994dec9aa9824879 2024-11-14T19:37:06,665 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/8eba0387e8484ff4994dec9aa9824879, entries=100, sequenceid=417, filesize=9.6 K 2024-11-14T19:37:06,666 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for ae3a414a536a107c2204f2532a580772 in 525ms, sequenceid=417, compaction requested=false 2024-11-14T19:37:06,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:06,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:06,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-14T19:37:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-14T19:37:06,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-14T19:37:06,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 680 msec 2024-11-14T19:37:06,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 685 msec 2024-11-14T19:37:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:06,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=154.31 KB heapSize=435.23 KB 2024-11-14T19:37:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T19:37:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T19:37:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-14T19:37:06,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/b00962a73ac142c48136d5a625a4a3b7 is 50, key is test_row_0/A:col10/1731613026512/Put/seqid=0 2024-11-14T19:37:06,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742320_1496 (size=14741) 2024-11-14T19:37:06,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613086853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613086854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613086855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613086855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613086856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613086964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613086964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613086973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613086973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613086973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,052 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/069848fc55e04f01ab8c865dc4082a7e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/069848fc55e04f01ab8c865dc4082a7e 2024-11-14T19:37:07,055 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into 069848fc55e04f01ab8c865dc4082a7e(size=12.8 K), total size for store is 22.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:07,055 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:07,055 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=12, startTime=1731613026139; duration=0sec 2024-11-14T19:37:07,055 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:07,055 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:37:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-14T19:37:07,124 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:07,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-14T19:37:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T19:37:07,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:07,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:07,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:07,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613087183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613087183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613087183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613087183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613087183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T19:37:07,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/b00962a73ac142c48136d5a625a4a3b7 2024-11-14T19:37:07,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ffbe5b78f2cc4abd91beb1fc7a1c2809 is 50, key is test_row_0/B:col10/1731613026512/Put/seqid=0 2024-11-14T19:37:07,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742321_1497 (size=12301) 2024-11-14T19:37:07,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T19:37:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T19:37:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:07,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T19:37:07,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613087503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613087503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613087503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613087503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613087503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:07,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T19:37:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ffbe5b78f2cc4abd91beb1fc7a1c2809 2024-11-14T19:37:07,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/18458f78472b4a35aeb9f52cf65b4bd9 is 50, key is test_row_0/C:col10/1731613026512/Put/seqid=0 2024-11-14T19:37:07,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742322_1498 (size=12301) 2024-11-14T19:37:07,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T19:37:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T19:37:07,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T19:37:07,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:07,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:08,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59016 deadline: 1731613088013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59048 deadline: 1731613088013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59008 deadline: 1731613088013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:08,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59070 deadline: 1731613088013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:08,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59056 deadline: 1731613088013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 , the old value is region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., hostname=867b237d0fa7,38601,1731612889637, seqNum=2 because the exception is null or not the one we care about 2024-11-14T19:37:08,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T19:37:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:08,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:08,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:08,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:08,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/18458f78472b4a35aeb9f52cf65b4bd9 2024-11-14T19:37:08,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/b00962a73ac142c48136d5a625a4a3b7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b00962a73ac142c48136d5a625a4a3b7 2024-11-14T19:37:08,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b00962a73ac142c48136d5a625a4a3b7, entries=200, sequenceid=474, filesize=14.4 K 2024-11-14T19:37:08,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ffbe5b78f2cc4abd91beb1fc7a1c2809 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ffbe5b78f2cc4abd91beb1fc7a1c2809 2024-11-14T19:37:08,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ffbe5b78f2cc4abd91beb1fc7a1c2809, entries=150, sequenceid=474, filesize=12.0 K 2024-11-14T19:37:08,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/18458f78472b4a35aeb9f52cf65b4bd9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18458f78472b4a35aeb9f52cf65b4bd9 2024-11-14T19:37:08,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18458f78472b4a35aeb9f52cf65b4bd9, entries=150, sequenceid=474, filesize=12.0 K 2024-11-14T19:37:08,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~435.19 KB/445632, currentSize=33.54 KB/34350 for ae3a414a536a107c2204f2532a580772 in 1248ms, sequenceid=474, compaction requested=true 2024-11-14T19:37:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:08,089 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:08,089 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3a414a536a107c2204f2532a580772:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:08,090 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:08,090 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/A is initiating minor compaction (all files) 2024-11-14T19:37:08,090 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:08,090 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/A in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:08,090 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/B is initiating minor compaction (all files) 2024-11-14T19:37:08,090 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/B in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:08,090 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2db4958b1bb74125932c629e4d8647e3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/7a50eb573a1b4a35ba977eb12a5d7928, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b00962a73ac142c48136d5a625a4a3b7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=36.8 K 2024-11-14T19:37:08,090 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/0782644511134e0b9a701eda95ca2a31, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ee05eb8686f943f58e78b9a31d474f35, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ffbe5b78f2cc4abd91beb1fc7a1c2809] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=34.5 K 2024-11-14T19:37:08,091 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db4958b1bb74125932c629e4d8647e3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731613024148 2024-11-14T19:37:08,091 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 0782644511134e0b9a701eda95ca2a31, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731613024148 2024-11-14T19:37:08,091 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a50eb573a1b4a35ba977eb12a5d7928, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731613025273 2024-11-14T19:37:08,091 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ee05eb8686f943f58e78b9a31d474f35, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731613025273 2024-11-14T19:37:08,091 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbe5b78f2cc4abd91beb1fc7a1c2809, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731613026499 2024-11-14T19:37:08,091 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting b00962a73ac142c48136d5a625a4a3b7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731613026499 2024-11-14T19:37:08,097 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:08,097 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#B#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:08,097 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/6d42205a88cb45debb88d5caba709b00 is 50, key is test_row_0/A:col10/1731613026512/Put/seqid=0 2024-11-14T19:37:08,098 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/459911ec0c0c4733a8d62a0922fdd971 is 50, key is test_row_0/B:col10/1731613026512/Put/seqid=0 2024-11-14T19:37:08,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742323_1499 (size=13221) 2024-11-14T19:37:08,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742324_1500 (size=13221) 2024-11-14T19:37:08,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-14T19:37:08,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:08,194 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T19:37:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:08,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/9dc342331a0c42858bead21cb2d77a9c is 50, key is test_row_1/A:col10/1731613026854/Put/seqid=0 2024-11-14T19:37:08,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742325_1501 (size=9857) 2024-11-14T19:37:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T19:37:08,510 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/6d42205a88cb45debb88d5caba709b00 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/6d42205a88cb45debb88d5caba709b00 2024-11-14T19:37:08,514 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/A of ae3a414a536a107c2204f2532a580772 into 6d42205a88cb45debb88d5caba709b00(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:08,514 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:08,514 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/459911ec0c0c4733a8d62a0922fdd971 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/459911ec0c0c4733a8d62a0922fdd971 2024-11-14T19:37:08,514 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/A, priority=13, startTime=1731613028089; duration=0sec 2024-11-14T19:37:08,514 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:08,514 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:A 2024-11-14T19:37:08,514 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:08,515 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:08,515 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): ae3a414a536a107c2204f2532a580772/C is initiating minor compaction (all files) 2024-11-14T19:37:08,515 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3a414a536a107c2204f2532a580772/C in TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:08,515 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/069848fc55e04f01ab8c865dc4082a7e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/8eba0387e8484ff4994dec9aa9824879, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18458f78472b4a35aeb9f52cf65b4bd9] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp, totalSize=34.5 K 2024-11-14T19:37:08,515 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 069848fc55e04f01ab8c865dc4082a7e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731613024148 2024-11-14T19:37:08,515 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eba0387e8484ff4994dec9aa9824879, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731613025273 2024-11-14T19:37:08,516 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18458f78472b4a35aeb9f52cf65b4bd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731613026499 2024-11-14T19:37:08,517 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/B of ae3a414a536a107c2204f2532a580772 into 459911ec0c0c4733a8d62a0922fdd971(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:08,517 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:08,517 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/B, priority=13, startTime=1731613028089; duration=0sec 2024-11-14T19:37:08,517 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:08,517 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:B 2024-11-14T19:37:08,523 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3a414a536a107c2204f2532a580772#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:08,523 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/d7f10d6ebb3648f99416edbc1e2cc542 is 50, key is test_row_0/C:col10/1731613026512/Put/seqid=0 2024-11-14T19:37:08,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742326_1502 (size=13221) 2024-11-14T19:37:08,602 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/9dc342331a0c42858bead21cb2d77a9c 2024-11-14T19:37:08,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ec748019eb2745bdb598bbac576a3864 is 50, key is test_row_1/B:col10/1731613026854/Put/seqid=0 2024-11-14T19:37:08,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742327_1503 (size=9857) 2024-11-14T19:37:08,612 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ec748019eb2745bdb598bbac576a3864 2024-11-14T19:37:08,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/08419ebac0104e23ad45f220e432395b is 50, key is test_row_1/C:col10/1731613026854/Put/seqid=0 2024-11-14T19:37:08,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742328_1504 (size=9857) 2024-11-14T19:37:08,800 INFO [Thread-1860 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1860. 2024-11-14T19:37:08,800 DEBUG [Thread-1860 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:08,800 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,801 DEBUG [Thread-1860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,801 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:08,802 INFO [Thread-1856 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1856. 2024-11-14T19:37:08,802 DEBUG [Thread-1856 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:08,802 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,802 DEBUG [Thread-1856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:08,803 INFO [Thread-1864 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1864. 2024-11-14T19:37:08,803 DEBUG [Thread-1864 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:08,803 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,803 DEBUG [Thread-1864 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,803 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:08,805 INFO [Thread-1862 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1862. 2024-11-14T19:37:08,805 DEBUG [Thread-1862 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:08,805 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,805 DEBUG [Thread-1862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:08,806 INFO [Thread-1858 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1858. 2024-11-14T19:37:08,806 DEBUG [Thread-1858 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:08,806 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,806 DEBUG [Thread-1858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:08,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:08,936 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/d7f10d6ebb3648f99416edbc1e2cc542 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d7f10d6ebb3648f99416edbc1e2cc542 2024-11-14T19:37:08,940 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3a414a536a107c2204f2532a580772/C of ae3a414a536a107c2204f2532a580772 into d7f10d6ebb3648f99416edbc1e2cc542(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:08,940 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:08,940 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., storeName=ae3a414a536a107c2204f2532a580772/C, priority=13, startTime=1731613028089; duration=0sec 2024-11-14T19:37:08,940 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:08,940 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3a414a536a107c2204f2532a580772:C 2024-11-14T19:37:09,025 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/08419ebac0104e23ad45f220e432395b 2024-11-14T19:37:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/9dc342331a0c42858bead21cb2d77a9c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/9dc342331a0c42858bead21cb2d77a9c 2024-11-14T19:37:09,036 INFO [Thread-1853 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1853. 2024-11-14T19:37:09,036 DEBUG [Thread-1853 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:09,036 DEBUG [Thread-1853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,036 DEBUG [Thread-1853 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:09,039 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/9dc342331a0c42858bead21cb2d77a9c, entries=100, sequenceid=483, filesize=9.6 K 2024-11-14T19:37:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/ec748019eb2745bdb598bbac576a3864 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ec748019eb2745bdb598bbac576a3864 2024-11-14T19:37:09,044 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ec748019eb2745bdb598bbac576a3864, entries=100, sequenceid=483, filesize=9.6 K 2024-11-14T19:37:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/08419ebac0104e23ad45f220e432395b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/08419ebac0104e23ad45f220e432395b 2024-11-14T19:37:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:09,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. as already flushing 2024-11-14T19:37:09,045 INFO [Thread-1849 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1849. 2024-11-14T19:37:09,045 INFO [Thread-1845 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1845. 2024-11-14T19:37:09,045 DEBUG [Thread-1849 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:09,045 INFO [Thread-1847 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1847. 2024-11-14T19:37:09,046 DEBUG [Thread-1845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:09,046 DEBUG [Thread-1849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,046 DEBUG [Thread-1847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:09,046 DEBUG [Thread-1845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,046 DEBUG [Thread-1847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,046 DEBUG [Thread-1849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,046 DEBUG [Thread-1847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,046 DEBUG [Thread-1845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:09,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:09,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:09,047 INFO [Thread-1851 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1851. 2024-11-14T19:37:09,047 DEBUG [Thread-1851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:09,047 DEBUG [Thread-1851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,047 DEBUG [Thread-1851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:09,050 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/08419ebac0104e23ad45f220e432395b, entries=100, sequenceid=483, filesize=9.6 K 2024-11-14T19:37:09,051 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=33.54 KB/34350 for ae3a414a536a107c2204f2532a580772 in 857ms, sequenceid=483, compaction requested=false 2024-11-14T19:37:09,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for ae3a414a536a107c2204f2532a580772: 2024-11-14T19:37:09,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:09,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-14T19:37:09,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-14T19:37:09,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-14T19:37:09,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9260 sec 2024-11-14T19:37:09,056 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.9310 sec 2024-11-14T19:37:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-14T19:37:09,284 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 936 2024-11-14T19:37:09,284 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2808 rows 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 943 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2829 rows 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 928 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2784 rows 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 942 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2788 rows 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 932 2024-11-14T19:37:09,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2796 rows 2024-11-14T19:37:09,285 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:37:09,285 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:37:09,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:09,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:09,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T19:37:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:09,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T19:37:09,291 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613029290"}]},"ts":"1731613029290"} 2024-11-14T19:37:09,292 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T19:37:09,292 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T19:37:09,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:37:09,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, UNASSIGN}] 2024-11-14T19:37:09,296 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, UNASSIGN 2024-11-14T19:37:09,296 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ae3a414a536a107c2204f2532a580772, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:09,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, UNASSIGN because future has completed 2024-11-14T19:37:09,298 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:37:09,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:37:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T19:37:09,451 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:09,451 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:37:09,451 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing ae3a414a536a107c2204f2532a580772, disabling compactions & flushes 2024-11-14T19:37:09,451 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:09,451 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:09,451 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. after waiting 0 ms 2024-11-14T19:37:09,451 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:09,451 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(2902): Flushing ae3a414a536a107c2204f2532a580772 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-14T19:37:09,452 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=A 2024-11-14T19:37:09,452 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:09,452 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=B 2024-11-14T19:37:09,452 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:09,452 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3a414a536a107c2204f2532a580772, store=C 2024-11-14T19:37:09,452 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:09,457 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/bd37ffe6aa564483a395be2f7c3aa62c is 50, key is test_row_0/A:col10/1731613029044/Put/seqid=0 2024-11-14T19:37:09,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742329_1505 (size=12301) 2024-11-14T19:37:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T19:37:09,863 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/bd37ffe6aa564483a395be2f7c3aa62c 2024-11-14T19:37:09,875 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/b329c60a61024b839b44278b35139eb1 is 50, key is test_row_0/B:col10/1731613029044/Put/seqid=0 2024-11-14T19:37:09,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742330_1506 (size=12301) 2024-11-14T19:37:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T19:37:10,279 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/b329c60a61024b839b44278b35139eb1 2024-11-14T19:37:10,291 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/11bc601becf34c008525d0d025813be2 is 50, key is test_row_0/C:col10/1731613029044/Put/seqid=0 2024-11-14T19:37:10,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742331_1507 (size=12301) 2024-11-14T19:37:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T19:37:10,696 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/11bc601becf34c008525d0d025813be2 2024-11-14T19:37:10,700 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/A/bd37ffe6aa564483a395be2f7c3aa62c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/bd37ffe6aa564483a395be2f7c3aa62c 2024-11-14T19:37:10,703 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/bd37ffe6aa564483a395be2f7c3aa62c, entries=150, sequenceid=494, filesize=12.0 K 2024-11-14T19:37:10,704 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/B/b329c60a61024b839b44278b35139eb1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b329c60a61024b839b44278b35139eb1 2024-11-14T19:37:10,707 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b329c60a61024b839b44278b35139eb1, entries=150, sequenceid=494, filesize=12.0 K 2024-11-14T19:37:10,708 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/.tmp/C/11bc601becf34c008525d0d025813be2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/11bc601becf34c008525d0d025813be2 2024-11-14T19:37:10,712 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/11bc601becf34c008525d0d025813be2, entries=150, sequenceid=494, filesize=12.0 K 2024-11-14T19:37:10,713 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ae3a414a536a107c2204f2532a580772 in 1262ms, sequenceid=494, compaction requested=true 2024-11-14T19:37:10,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b61dc17042cd4cbfbd1d50d252399580, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a3e8321017d744b4a500c4c6b638bf8a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/24ee807c9c0e467ab778a2d0b5ee9088, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c26b9929763e423d9623c2b34a6d7617, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/36daa09f1b14484cabb763835a5419c7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/1778555eeb144e4597777ea4d42d8d4e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c7fb3058806a44acb21212ee089d69fe, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/86b30276f11745008ee37f41bdc5bd5e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ea04a3fcc4c419284d65e1e533d2a0e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/f97dd49161ea4038a23997bc2d4dcf7c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/eb26f4cc272d40e0a1253e4b3e1c0220, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ae706c3548345fd8a19eead473f29d8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2f442e3d10c94ea288a2acc20905a324, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0dcd1f74c55f40ceb577decc885c939d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/02cf717e3b3f4d03bfb9e26e84253935, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/621844d70d724721ade696808f63a055, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/749731a3e3094d0f9f159b5bb1fe0435, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8d33ec1cd49b4689b0f0b736563fc95f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/d6fbe60a43a44dbab50d91865bbc889f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/634055fda26f4f7ebe88dbd6f25178a4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/cf5a4f917f8340bf92eb3062b0657a92, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/ec9dc6b161e748f9b358b30b0ba292c2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0e03e01c07644b628ef7693cfd95d018, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/375127486cb64bf98ed24d463ece4392, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2db4958b1bb74125932c629e4d8647e3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/7a50eb573a1b4a35ba977eb12a5d7928, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b00962a73ac142c48136d5a625a4a3b7] to archive 2024-11-14T19:37:10,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:37:10,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b61dc17042cd4cbfbd1d50d252399580 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b61dc17042cd4cbfbd1d50d252399580 2024-11-14T19:37:10,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a3e8321017d744b4a500c4c6b638bf8a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/a3e8321017d744b4a500c4c6b638bf8a 2024-11-14T19:37:10,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/24ee807c9c0e467ab778a2d0b5ee9088 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/24ee807c9c0e467ab778a2d0b5ee9088 2024-11-14T19:37:10,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c26b9929763e423d9623c2b34a6d7617 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c26b9929763e423d9623c2b34a6d7617 2024-11-14T19:37:10,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/36daa09f1b14484cabb763835a5419c7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/36daa09f1b14484cabb763835a5419c7 2024-11-14T19:37:10,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/1778555eeb144e4597777ea4d42d8d4e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/1778555eeb144e4597777ea4d42d8d4e 2024-11-14T19:37:10,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c7fb3058806a44acb21212ee089d69fe to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/c7fb3058806a44acb21212ee089d69fe 2024-11-14T19:37:10,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/86b30276f11745008ee37f41bdc5bd5e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/86b30276f11745008ee37f41bdc5bd5e 2024-11-14T19:37:10,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ea04a3fcc4c419284d65e1e533d2a0e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ea04a3fcc4c419284d65e1e533d2a0e 2024-11-14T19:37:10,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/f97dd49161ea4038a23997bc2d4dcf7c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/f97dd49161ea4038a23997bc2d4dcf7c 2024-11-14T19:37:10,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/eb26f4cc272d40e0a1253e4b3e1c0220 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/eb26f4cc272d40e0a1253e4b3e1c0220 2024-11-14T19:37:10,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ae706c3548345fd8a19eead473f29d8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8ae706c3548345fd8a19eead473f29d8 2024-11-14T19:37:10,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2f442e3d10c94ea288a2acc20905a324 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2f442e3d10c94ea288a2acc20905a324 2024-11-14T19:37:10,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0dcd1f74c55f40ceb577decc885c939d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0dcd1f74c55f40ceb577decc885c939d 2024-11-14T19:37:10,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/02cf717e3b3f4d03bfb9e26e84253935 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/02cf717e3b3f4d03bfb9e26e84253935 2024-11-14T19:37:10,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/621844d70d724721ade696808f63a055 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/621844d70d724721ade696808f63a055 2024-11-14T19:37:10,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/749731a3e3094d0f9f159b5bb1fe0435 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/749731a3e3094d0f9f159b5bb1fe0435 2024-11-14T19:37:10,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8d33ec1cd49b4689b0f0b736563fc95f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/8d33ec1cd49b4689b0f0b736563fc95f 2024-11-14T19:37:10,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/d6fbe60a43a44dbab50d91865bbc889f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/d6fbe60a43a44dbab50d91865bbc889f 2024-11-14T19:37:10,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/634055fda26f4f7ebe88dbd6f25178a4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/634055fda26f4f7ebe88dbd6f25178a4 2024-11-14T19:37:10,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/cf5a4f917f8340bf92eb3062b0657a92 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/cf5a4f917f8340bf92eb3062b0657a92 2024-11-14T19:37:10,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/ec9dc6b161e748f9b358b30b0ba292c2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/ec9dc6b161e748f9b358b30b0ba292c2 2024-11-14T19:37:10,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0e03e01c07644b628ef7693cfd95d018 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/0e03e01c07644b628ef7693cfd95d018 2024-11-14T19:37:10,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/375127486cb64bf98ed24d463ece4392 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/375127486cb64bf98ed24d463ece4392 2024-11-14T19:37:10,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2db4958b1bb74125932c629e4d8647e3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/2db4958b1bb74125932c629e4d8647e3 2024-11-14T19:37:10,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/7a50eb573a1b4a35ba977eb12a5d7928 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/7a50eb573a1b4a35ba977eb12a5d7928 2024-11-14T19:37:10,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b00962a73ac142c48136d5a625a4a3b7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/b00962a73ac142c48136d5a625a4a3b7 2024-11-14T19:37:10,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5598e75bfb004163bbb04f396077f465, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/17ab52a43c6f4c81a6824e7aa276914f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/d11975e24a7646cf8c7f9e747a023e87, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5ee11a6ca85b4b158418b51a1e97ab6f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3a4ad9167a944c6591e3a0b32c8ca099, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/7573533b1fc74f938dc8472a3890480d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/81e06dbe576746abb783b9485bf996bc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/cd98c8b2b0764b7d9bb77a01ae045dc8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3569e1f2034844b8bc78185592ff0bd9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/79b191a7216e40caa0b7a2808ae792fb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/452f534a24244ebb9f5fa6190abea761, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/15814acbebfe48148c2e73cf22813afb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/a8dd56c731bc4165928020965cf6f094, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31d921ee4a414e26befd064d41dbd9c4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31093080d4bb41cea7ac6c4e9d351825, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/2c0e6eaea3e0486faa9a2ccc6b656cc4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/eef26ae77aa9482eafba36dbfbba70e6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/74ad0e77c2054e5aa3eb3ab178df3531, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1104b10f63d84c5fad0690f56a29111b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/803f7b7cdae444618554d83bddb6ff8c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3cf8c7e990534affa7a8a8bb76d37db4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ae3a77d69b24d0aa16942208b448b41, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ba3446a6677431880e7d02d2c9c847e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/0782644511134e0b9a701eda95ca2a31, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/91a8484aca654cb89588cf7423dcba5a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ee05eb8686f943f58e78b9a31d474f35, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ffbe5b78f2cc4abd91beb1fc7a1c2809] to archive 2024-11-14T19:37:10,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:37:10,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5598e75bfb004163bbb04f396077f465 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5598e75bfb004163bbb04f396077f465 2024-11-14T19:37:10,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/17ab52a43c6f4c81a6824e7aa276914f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/17ab52a43c6f4c81a6824e7aa276914f 2024-11-14T19:37:10,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/d11975e24a7646cf8c7f9e747a023e87 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/d11975e24a7646cf8c7f9e747a023e87 2024-11-14T19:37:10,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5ee11a6ca85b4b158418b51a1e97ab6f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/5ee11a6ca85b4b158418b51a1e97ab6f 2024-11-14T19:37:10,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3a4ad9167a944c6591e3a0b32c8ca099 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3a4ad9167a944c6591e3a0b32c8ca099 2024-11-14T19:37:10,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/7573533b1fc74f938dc8472a3890480d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/7573533b1fc74f938dc8472a3890480d 2024-11-14T19:37:10,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/81e06dbe576746abb783b9485bf996bc to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/81e06dbe576746abb783b9485bf996bc 2024-11-14T19:37:10,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/cd98c8b2b0764b7d9bb77a01ae045dc8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/cd98c8b2b0764b7d9bb77a01ae045dc8 2024-11-14T19:37:10,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3569e1f2034844b8bc78185592ff0bd9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3569e1f2034844b8bc78185592ff0bd9 2024-11-14T19:37:10,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/79b191a7216e40caa0b7a2808ae792fb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/79b191a7216e40caa0b7a2808ae792fb 2024-11-14T19:37:10,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/452f534a24244ebb9f5fa6190abea761 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/452f534a24244ebb9f5fa6190abea761 2024-11-14T19:37:10,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/15814acbebfe48148c2e73cf22813afb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/15814acbebfe48148c2e73cf22813afb 2024-11-14T19:37:10,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/a8dd56c731bc4165928020965cf6f094 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/a8dd56c731bc4165928020965cf6f094 2024-11-14T19:37:10,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31d921ee4a414e26befd064d41dbd9c4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31d921ee4a414e26befd064d41dbd9c4 2024-11-14T19:37:10,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31093080d4bb41cea7ac6c4e9d351825 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/31093080d4bb41cea7ac6c4e9d351825 2024-11-14T19:37:10,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/2c0e6eaea3e0486faa9a2ccc6b656cc4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/2c0e6eaea3e0486faa9a2ccc6b656cc4 2024-11-14T19:37:10,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/eef26ae77aa9482eafba36dbfbba70e6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/eef26ae77aa9482eafba36dbfbba70e6 2024-11-14T19:37:10,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/74ad0e77c2054e5aa3eb3ab178df3531 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/74ad0e77c2054e5aa3eb3ab178df3531 2024-11-14T19:37:10,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1104b10f63d84c5fad0690f56a29111b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/1104b10f63d84c5fad0690f56a29111b 2024-11-14T19:37:10,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/803f7b7cdae444618554d83bddb6ff8c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/803f7b7cdae444618554d83bddb6ff8c 2024-11-14T19:37:10,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3cf8c7e990534affa7a8a8bb76d37db4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3cf8c7e990534affa7a8a8bb76d37db4 2024-11-14T19:37:10,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ae3a77d69b24d0aa16942208b448b41 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ae3a77d69b24d0aa16942208b448b41 2024-11-14T19:37:10,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ba3446a6677431880e7d02d2c9c847e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/3ba3446a6677431880e7d02d2c9c847e 2024-11-14T19:37:10,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/0782644511134e0b9a701eda95ca2a31 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/0782644511134e0b9a701eda95ca2a31 2024-11-14T19:37:10,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/91a8484aca654cb89588cf7423dcba5a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/91a8484aca654cb89588cf7423dcba5a 2024-11-14T19:37:10,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ee05eb8686f943f58e78b9a31d474f35 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ee05eb8686f943f58e78b9a31d474f35 2024-11-14T19:37:10,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ffbe5b78f2cc4abd91beb1fc7a1c2809 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ffbe5b78f2cc4abd91beb1fc7a1c2809 2024-11-14T19:37:10,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/3d64a382d0f84a5cb31cc90e0bb1844b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/79153cb11e414b5184bd4b663f65b976, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a396fe54f28c42f6ab1ae72bd0ee5a29, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/9fdeb7c2554e46be95eb8455d7b8ab7c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/501489ca0fcd43ffa99735dd771a4eee, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/267d0e4cb39044c48aad979ba2c857f1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18cfa11c6dcd4142b2453259ca3f5bb6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/46c09534787c4b1ea629f2c8f21910b3, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6bbd481eb7bc4c64b3eff5e5a60d5d90, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/90647a67de5d4617b60be161b1a6c473, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c4af45ece76f4a69a5f0f7ab52d65b74, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/179793d8f027458abb44cbee7f5e1c8d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6ab6cd1a2a7c40c586f7d42cd33c595c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e46a50efe3df47979c77db3050e7f6dd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c06ff679ca1e4dc6bd040ba5299a40e9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/100646b109dd4eef855a8b84541fda37, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/386a5d0cbd2545dcb1008c8b70f5cfae, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/942ccceea330467490fdb44e231cca62, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/4f85f12f58494883949e8e28464fb9f7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e896ba48b5534a06b269910b8b272c2c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/f5273920b58b42138433b62cdfd9640a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/fca4b3780c30486faa5d221dede682bf, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/48abad874da44bd0aeb9395c62581b7c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/069848fc55e04f01ab8c865dc4082a7e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/38f2992e7cff41a2922307e9a6f4b8d5, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/8eba0387e8484ff4994dec9aa9824879, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18458f78472b4a35aeb9f52cf65b4bd9] to archive 2024-11-14T19:37:10,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:37:10,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/3d64a382d0f84a5cb31cc90e0bb1844b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/3d64a382d0f84a5cb31cc90e0bb1844b 2024-11-14T19:37:10,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/79153cb11e414b5184bd4b663f65b976 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/79153cb11e414b5184bd4b663f65b976 2024-11-14T19:37:10,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a396fe54f28c42f6ab1ae72bd0ee5a29 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/a396fe54f28c42f6ab1ae72bd0ee5a29 2024-11-14T19:37:10,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/9fdeb7c2554e46be95eb8455d7b8ab7c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/9fdeb7c2554e46be95eb8455d7b8ab7c 2024-11-14T19:37:10,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/501489ca0fcd43ffa99735dd771a4eee to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/501489ca0fcd43ffa99735dd771a4eee 2024-11-14T19:37:10,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/267d0e4cb39044c48aad979ba2c857f1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/267d0e4cb39044c48aad979ba2c857f1 2024-11-14T19:37:10,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18cfa11c6dcd4142b2453259ca3f5bb6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18cfa11c6dcd4142b2453259ca3f5bb6 2024-11-14T19:37:10,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/46c09534787c4b1ea629f2c8f21910b3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/46c09534787c4b1ea629f2c8f21910b3 2024-11-14T19:37:10,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6bbd481eb7bc4c64b3eff5e5a60d5d90 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6bbd481eb7bc4c64b3eff5e5a60d5d90 2024-11-14T19:37:10,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/90647a67de5d4617b60be161b1a6c473 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/90647a67de5d4617b60be161b1a6c473 2024-11-14T19:37:10,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c4af45ece76f4a69a5f0f7ab52d65b74 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c4af45ece76f4a69a5f0f7ab52d65b74 2024-11-14T19:37:10,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/179793d8f027458abb44cbee7f5e1c8d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/179793d8f027458abb44cbee7f5e1c8d 2024-11-14T19:37:10,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6ab6cd1a2a7c40c586f7d42cd33c595c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/6ab6cd1a2a7c40c586f7d42cd33c595c 2024-11-14T19:37:10,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e46a50efe3df47979c77db3050e7f6dd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e46a50efe3df47979c77db3050e7f6dd 2024-11-14T19:37:10,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c06ff679ca1e4dc6bd040ba5299a40e9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/c06ff679ca1e4dc6bd040ba5299a40e9 2024-11-14T19:37:10,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/100646b109dd4eef855a8b84541fda37 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/100646b109dd4eef855a8b84541fda37 2024-11-14T19:37:10,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/386a5d0cbd2545dcb1008c8b70f5cfae to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/386a5d0cbd2545dcb1008c8b70f5cfae 2024-11-14T19:37:10,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/942ccceea330467490fdb44e231cca62 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/942ccceea330467490fdb44e231cca62 2024-11-14T19:37:10,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/4f85f12f58494883949e8e28464fb9f7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/4f85f12f58494883949e8e28464fb9f7 2024-11-14T19:37:10,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e896ba48b5534a06b269910b8b272c2c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/e896ba48b5534a06b269910b8b272c2c 2024-11-14T19:37:10,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/f5273920b58b42138433b62cdfd9640a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/f5273920b58b42138433b62cdfd9640a 2024-11-14T19:37:10,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/fca4b3780c30486faa5d221dede682bf to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/fca4b3780c30486faa5d221dede682bf 2024-11-14T19:37:10,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/48abad874da44bd0aeb9395c62581b7c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/48abad874da44bd0aeb9395c62581b7c 2024-11-14T19:37:10,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/069848fc55e04f01ab8c865dc4082a7e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/069848fc55e04f01ab8c865dc4082a7e 2024-11-14T19:37:10,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/38f2992e7cff41a2922307e9a6f4b8d5 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/38f2992e7cff41a2922307e9a6f4b8d5 2024-11-14T19:37:10,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/8eba0387e8484ff4994dec9aa9824879 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/8eba0387e8484ff4994dec9aa9824879 2024-11-14T19:37:10,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18458f78472b4a35aeb9f52cf65b4bd9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/18458f78472b4a35aeb9f52cf65b4bd9 2024-11-14T19:37:10,827 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/recovered.edits/497.seqid, newMaxSeqId=497, maxSeqId=1 2024-11-14T19:37:10,827 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. 2024-11-14T19:37:10,827 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for ae3a414a536a107c2204f2532a580772: Waiting for close lock at 1731613029451Running coprocessor pre-close hooks at 1731613029451Disabling compacts and flushes for region at 1731613029451Disabling writes for close at 1731613029451Obtaining lock to block concurrent updates at 1731613029452 (+1 ms)Preparing flush snapshotting stores in ae3a414a536a107c2204f2532a580772 at 1731613029452Finished memstore snapshotting TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731613029452Flushing stores of TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772. at 1731613029453 (+1 ms)Flushing ae3a414a536a107c2204f2532a580772/A: creating writer at 1731613029453Flushing ae3a414a536a107c2204f2532a580772/A: appending metadata at 1731613029457 (+4 ms)Flushing ae3a414a536a107c2204f2532a580772/A: closing flushed file at 1731613029457Flushing ae3a414a536a107c2204f2532a580772/B: creating writer at 1731613029870 (+413 ms)Flushing ae3a414a536a107c2204f2532a580772/B: appending metadata at 1731613029874 (+4 ms)Flushing ae3a414a536a107c2204f2532a580772/B: closing flushed file at 1731613029874Flushing ae3a414a536a107c2204f2532a580772/C: creating writer at 1731613030286 (+412 ms)Flushing ae3a414a536a107c2204f2532a580772/C: appending metadata at 1731613030290 (+4 ms)Flushing ae3a414a536a107c2204f2532a580772/C: closing flushed file at 1731613030290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c7c9685: reopening flushed file at 1731613030699 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e898d6e: reopening flushed file at 1731613030703 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@536d7f7b: reopening flushed file at 1731613030707 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ae3a414a536a107c2204f2532a580772 in 1262ms, sequenceid=494, compaction requested=true at 1731613030713 (+6 ms)Writing region close event to WAL at 1731613030824 (+111 ms)Running coprocessor post-close hooks at 1731613030827 (+3 ms)Closed at 1731613030827 2024-11-14T19:37:10,829 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:10,829 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=ae3a414a536a107c2204f2532a580772, regionState=CLOSED 2024-11-14T19:37:10,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:37:10,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-14T19:37:10,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure ae3a414a536a107c2204f2532a580772, server=867b237d0fa7,38601,1731612889637 in 1.5330 sec 2024-11-14T19:37:10,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-14T19:37:10,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3a414a536a107c2204f2532a580772, UNASSIGN in 1.5370 sec 2024-11-14T19:37:10,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-14T19:37:10,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5410 sec 2024-11-14T19:37:10,837 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613030837"}]},"ts":"1731613030837"} 2024-11-14T19:37:10,838 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T19:37:10,838 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T19:37:10,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5520 sec 2024-11-14T19:37:11,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-14T19:37:11,445 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:11,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T19:37:11,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:11,453 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=139, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:11,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T19:37:11,455 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=139, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:11,458 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:11,461 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/recovered.edits] 2024-11-14T19:37:11,464 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/6d42205a88cb45debb88d5caba709b00 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/6d42205a88cb45debb88d5caba709b00 2024-11-14T19:37:11,466 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/9dc342331a0c42858bead21cb2d77a9c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/9dc342331a0c42858bead21cb2d77a9c 2024-11-14T19:37:11,467 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/bd37ffe6aa564483a395be2f7c3aa62c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/A/bd37ffe6aa564483a395be2f7c3aa62c 2024-11-14T19:37:11,469 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/459911ec0c0c4733a8d62a0922fdd971 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/459911ec0c0c4733a8d62a0922fdd971 2024-11-14T19:37:11,470 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b329c60a61024b839b44278b35139eb1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/b329c60a61024b839b44278b35139eb1 2024-11-14T19:37:11,471 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ec748019eb2745bdb598bbac576a3864 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/B/ec748019eb2745bdb598bbac576a3864 2024-11-14T19:37:11,472 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/08419ebac0104e23ad45f220e432395b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/08419ebac0104e23ad45f220e432395b 2024-11-14T19:37:11,473 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/11bc601becf34c008525d0d025813be2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/11bc601becf34c008525d0d025813be2 2024-11-14T19:37:11,474 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d7f10d6ebb3648f99416edbc1e2cc542 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/C/d7f10d6ebb3648f99416edbc1e2cc542 2024-11-14T19:37:11,477 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/recovered.edits/497.seqid to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772/recovered.edits/497.seqid 2024-11-14T19:37:11,477 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/ae3a414a536a107c2204f2532a580772 2024-11-14T19:37:11,477 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T19:37:11,479 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=139, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:11,481 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T19:37:11,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T19:37:11,484 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=139, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:11,484 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T19:37:11,484 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731613031484"}]},"ts":"9223372036854775807"} 2024-11-14T19:37:11,487 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T19:37:11,487 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae3a414a536a107c2204f2532a580772, NAME => 'TestAcidGuarantees,,1731613007576.ae3a414a536a107c2204f2532a580772.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T19:37:11,487 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T19:37:11,487 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731613031487"}]},"ts":"9223372036854775807"} 2024-11-14T19:37:11,489 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T19:37:11,490 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=139, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:11,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-14T19:37:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-14T19:37:11,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T19:37:11,567 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:11,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=599 (was 572) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5452 (was 5498) 2024-11-14T19:37:11,582 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=599, ProcessCount=11, AvailableMemoryMB=5452 2024-11-14T19:37:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:37:11,584 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:37:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:11,586 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T19:37:11,586 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:11,586 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 140 2024-11-14T19:37:11,587 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T19:37:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T19:37:11,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742332_1508 (size=1061) 2024-11-14T19:37:11,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T19:37:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T19:37:11,994 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041 2024-11-14T19:37:12,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742333_1509 (size=53) 2024-11-14T19:37:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T19:37:12,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:37:12,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 12ed89d0dac4353a0596c5193935d87c, disabling compactions & flushes 2024-11-14T19:37:12,403 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. after waiting 0 ms 2024-11-14T19:37:12,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,404 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 12ed89d0dac4353a0596c5193935d87c: Waiting for close lock at 1731613032403Disabling compacts and flushes for region at 1731613032403Disabling writes for close at 1731613032404 (+1 ms)Writing region close event to WAL at 1731613032404Closed at 1731613032404 2024-11-14T19:37:12,407 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T19:37:12,408 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731613032407"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731613032407"}]},"ts":"1731613032407"} 2024-11-14T19:37:12,410 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T19:37:12,411 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T19:37:12,411 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613032411"}]},"ts":"1731613032411"} 2024-11-14T19:37:12,413 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-14T19:37:12,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, ASSIGN}] 2024-11-14T19:37:12,415 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, ASSIGN 2024-11-14T19:37:12,416 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, ASSIGN; state=OFFLINE, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=false 2024-11-14T19:37:12,567 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:12,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, ASSIGN because future has completed 2024-11-14T19:37:12,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:37:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T19:37:12,730 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,730 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7752): Opening region: {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:37:12,731 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,731 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:37:12,731 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7794): checking encryption for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,731 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(7797): checking classloading for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,733 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,736 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:37:12,736 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12ed89d0dac4353a0596c5193935d87c columnFamilyName A 2024-11-14T19:37:12,737 DEBUG [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:12,738 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(327): Store=12ed89d0dac4353a0596c5193935d87c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:37:12,738 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,740 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:37:12,740 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12ed89d0dac4353a0596c5193935d87c columnFamilyName B 2024-11-14T19:37:12,740 DEBUG [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:12,741 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(327): Store=12ed89d0dac4353a0596c5193935d87c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:37:12,741 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,742 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:37:12,742 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12ed89d0dac4353a0596c5193935d87c columnFamilyName C 2024-11-14T19:37:12,742 DEBUG [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:12,743 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(327): Store=12ed89d0dac4353a0596c5193935d87c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:37:12,743 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,743 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1038): replaying wal for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,744 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,744 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,746 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1048): stopping wal replay for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,746 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1060): Cleaning up temporary data for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,746 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:37:12,747 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1093): writing seq id for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,749 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T19:37:12,749 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1114): Opened 12ed89d0dac4353a0596c5193935d87c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63824880, jitterRate=-0.0489351749420166}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:37:12,749 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:12,749 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegion(1006): Region open journal for 12ed89d0dac4353a0596c5193935d87c: Running coprocessor pre-open hook at 1731613032731Writing region info on filesystem at 1731613032731Initializing all the Stores at 1731613032732 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613032733 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613032733Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613032733Cleaning up temporary data from old regions at 1731613032746 (+13 ms)Running coprocessor post-open hooks at 1731613032749 (+3 ms)Region opened successfully at 1731613032749 2024-11-14T19:37:12,750 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., pid=142, masterSystemTime=1731613032725 2024-11-14T19:37:12,751 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,751 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=142}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:12,752 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=OPEN, openSeqNum=2, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:12,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:37:12,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-14T19:37:12,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 in 181 msec 2024-11-14T19:37:12,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-14T19:37:12,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, ASSIGN in 342 msec 2024-11-14T19:37:12,757 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T19:37:12,757 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613032757"}]},"ts":"1731613032757"} 2024-11-14T19:37:12,758 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-14T19:37:12,758 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=140, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T19:37:12,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-14T19:37:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-14T19:37:13,735 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:13,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@724ae149, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:13,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:13,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:13,740 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae1d311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:13,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c698726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:13,744 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:13,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:13,747 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T19:37:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 867b237d0fa7,38019,1731612888274 2024-11-14T19:37:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@218634de 2024-11-14T19:37:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T19:37:13,749 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T19:37:13,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-14T19:37:13,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T19:37:13,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:13,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742334_1510 (size=1097) 2024-11-14T19:37:14,169 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-14T19:37:14,169 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-14T19:37:14,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:37:14,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, REOPEN/MOVE}] 2024-11-14T19:37:14,183 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, REOPEN/MOVE 2024-11-14T19:37:14,184 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, REOPEN/MOVE because future has completed 2024-11-14T19:37:14,188 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:37:14,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:37:14,343 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,343 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:37:14,343 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 12ed89d0dac4353a0596c5193935d87c, disabling compactions & flushes 2024-11-14T19:37:14,343 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,343 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,343 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. after waiting 0 ms 2024-11-14T19:37:14,344 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,351 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-14T19:37:14,352 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,352 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 12ed89d0dac4353a0596c5193935d87c: Waiting for close lock at 1731613034343Running coprocessor pre-close hooks at 1731613034343Disabling compacts and flushes for region at 1731613034343Disabling writes for close at 1731613034343Writing region close event to WAL at 1731613034344 (+1 ms)Running coprocessor post-close hooks at 1731613034352 (+8 ms)Closed at 1731613034352 2024-11-14T19:37:14,352 WARN [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionServer(3299): Not adding moved region record: 12ed89d0dac4353a0596c5193935d87c to self. 2024-11-14T19:37:14,354 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,356 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=CLOSED 2024-11-14T19:37:14,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:37:14,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-14T19:37:14,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 in 171 msec 2024-11-14T19:37:14,363 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, REOPEN/MOVE; state=CLOSED, location=867b237d0fa7,38601,1731612889637; forceNewPlan=false, retain=true 2024-11-14T19:37:14,514 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=OPENING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, REOPEN/MOVE because future has completed 2024-11-14T19:37:14,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:37:14,677 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,677 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7752): Opening region: {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T19:37:14,678 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,678 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T19:37:14,679 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7794): checking encryption for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,679 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(7797): checking classloading for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,682 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,684 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:37:14,684 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12ed89d0dac4353a0596c5193935d87c columnFamilyName A 2024-11-14T19:37:14,686 DEBUG [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:14,686 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(327): Store=12ed89d0dac4353a0596c5193935d87c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:37:14,687 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,687 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:37:14,687 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12ed89d0dac4353a0596c5193935d87c columnFamilyName B 2024-11-14T19:37:14,687 DEBUG [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:14,687 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(327): Store=12ed89d0dac4353a0596c5193935d87c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:37:14,687 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,688 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-14T19:37:14,688 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12ed89d0dac4353a0596c5193935d87c columnFamilyName C 2024-11-14T19:37:14,688 DEBUG [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:14,688 INFO [StoreOpener-12ed89d0dac4353a0596c5193935d87c-1 {}] regionserver.HStore(327): Store=12ed89d0dac4353a0596c5193935d87c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T19:37:14,689 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,689 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1038): replaying wal for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,689 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,690 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,691 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1048): stopping wal replay for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,691 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1060): Cleaning up temporary data for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,691 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T19:37:14,692 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1093): writing seq id for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,693 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1114): Opened 12ed89d0dac4353a0596c5193935d87c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74663296, jitterRate=0.11256980895996094}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T19:37:14,693 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,693 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegion(1006): Region open journal for 12ed89d0dac4353a0596c5193935d87c: Running coprocessor pre-open hook at 1731613034679Writing region info on filesystem at 1731613034679Initializing all the Stores at 1731613034681 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613034681Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613034682 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731613034682Cleaning up temporary data from old regions at 1731613034691 (+9 ms)Running coprocessor post-open hooks at 1731613034693 (+2 ms)Region opened successfully at 1731613034693 2024-11-14T19:37:14,694 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., pid=147, masterSystemTime=1731613034670 2024-11-14T19:37:14,695 DEBUG [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,695 INFO [RS_OPEN_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_OPEN_REGION, pid=147}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,696 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=OPEN, openSeqNum=5, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:37:14,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=145 2024-11-14T19:37:14,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 in 180 msec 2024-11-14T19:37:14,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-14T19:37:14,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, REOPEN/MOVE in 518 msec 2024-11-14T19:37:14,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-14T19:37:14,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-11-14T19:37:14,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 952 msec 2024-11-14T19:37:14,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-14T19:37:14,707 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:14,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b9d0221, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700b116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f0a708, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bc30a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,711 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,711 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,711 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77de7c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58b18374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3876a305, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6616e6f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49706e9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6266891e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5839019f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e0dd3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79538ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,721 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa537f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,722 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d451d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32aa3c74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,724 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a432c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4ef569, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d16a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,728 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53afbea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,729 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1360d0b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16487325, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,731 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bd41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,733 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6048da80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5129a55e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,735 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349a613f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,736 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140b73de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f0ee4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 867b237d0fa7,38019,-1 for getting cluster id 2024-11-14T19:37:14,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T19:37:14,738 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e950080f-d9c0-4d61-bb4c-96168b3f7750' 2024-11-14T19:37:14,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T19:37:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e950080f-d9c0-4d61-bb4c-96168b3f7750" 2024-11-14T19:37:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26d438d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [867b237d0fa7,38019,-1] 2024-11-14T19:37:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T19:37:14,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:14,739 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T19:37:14,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233a9464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T19:37:14,740 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,740 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,740 DEBUG [Thread-2281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,740 DEBUG [Thread-2300 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,740 DEBUG [Thread-2294 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,741 DEBUG [Thread-2298 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,741 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,741 DEBUG [Thread-2292 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,741 DEBUG [Thread-2296 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,743 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,743 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T19:37:14,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,744 DEBUG [Thread-2285 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=867b237d0fa7,38601,1731612889637, seqNum=-1] 2024-11-14T19:37:14,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,744 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:14,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,744 DEBUG [Thread-2285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T19:37:14,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T19:37:14,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-14T19:37:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T19:37:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,748 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5] 2024-11-14T19:37:14,748 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:14,749 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:14,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:14,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:14,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:37:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:14,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731613034787, max timestamp=1731613034796Num uniques 859; 2024-11-14T19:37:14,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731613034787, max timestamp=1731613034796Num uniques 859; 2024-11-14T19:37:14,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:37:14,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:37:14,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731613034787, max timestamp=1731613034796Num uniques 859; 2024-11-14T19:37:14,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613094795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:37:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114982f45a84bfe44a9a5c87ba3aac21524_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613034777/Put/seqid=0 2024-11-14T19:37:14,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742335_1511 (size=14594) 2024-11-14T19:37:14,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613094815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613094844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613094844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613094844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T19:37:14,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:14,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:14,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:14,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613094903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613094953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613094953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613094953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:14,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613094953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:14,955 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:15,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:15,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T19:37:15,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613095113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,116 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613095163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613095163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613095163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613095163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,167 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:15,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:15,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,212 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:15,215 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114982f45a84bfe44a9a5c87ba3aac21524_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114982f45a84bfe44a9a5c87ba3aac21524_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:15,216 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5c4f35696ebb42e1877d261868d2c2db, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:15,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5c4f35696ebb42e1877d261868d2c2db is 175, key is test_row_0/A:col10/1731613034777/Put/seqid=0 2024-11-14T19:37:15,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742336_1512 (size=39549) 2024-11-14T19:37:15,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T19:37:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613095423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613095473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613095473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613095473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613095473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:15,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:15,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,621 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5c4f35696ebb42e1877d261868d2c2db 2024-11-14T19:37:15,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/76250cbadf14464e83f4663dd2871b5c is 50, key is test_row_0/B:col10/1731613034777/Put/seqid=0 2024-11-14T19:37:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742337_1513 (size=12001) 2024-11-14T19:37:15,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/76250cbadf14464e83f4663dd2871b5c 2024-11-14T19:37:15,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/8397fa4eef1a474991529708f892fa0e is 50, key is test_row_0/C:col10/1731613034777/Put/seqid=0 2024-11-14T19:37:15,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742338_1514 (size=12001) 2024-11-14T19:37:15,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T19:37:15,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613095933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:15,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:15,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613095983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613095984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613095984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613095993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:16,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:16,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:16,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:16,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:16,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:16,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:16,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/8397fa4eef1a474991529708f892fa0e 2024-11-14T19:37:16,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5c4f35696ebb42e1877d261868d2c2db as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5c4f35696ebb42e1877d261868d2c2db 2024-11-14T19:37:16,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5c4f35696ebb42e1877d261868d2c2db, entries=200, sequenceid=14, filesize=38.6 K 2024-11-14T19:37:16,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/76250cbadf14464e83f4663dd2871b5c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/76250cbadf14464e83f4663dd2871b5c 2024-11-14T19:37:16,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/76250cbadf14464e83f4663dd2871b5c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T19:37:16,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/8397fa4eef1a474991529708f892fa0e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8397fa4eef1a474991529708f892fa0e 2024-11-14T19:37:16,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8397fa4eef1a474991529708f892fa0e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-14T19:37:16,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 12ed89d0dac4353a0596c5193935d87c in 1454ms, sequenceid=14, compaction requested=false 2024-11-14T19:37:16,233 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-14T19:37:16,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:16,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-14T19:37:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:16,284 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-14T19:37:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:16,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3a7894e3c4242de8922d895cdc7c6a3_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613034826/Put/seqid=0 2024-11-14T19:37:16,360 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T19:37:16,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742339_1515 (size=12154) 2024-11-14T19:37:16,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:16,790 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b3a7894e3c4242de8922d895cdc7c6a3_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3a7894e3c4242de8922d895cdc7c6a3_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:16,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/efb57c24e3b24e9caec56be7ffdf757a, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:16,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/efb57c24e3b24e9caec56be7ffdf757a is 175, key is test_row_0/A:col10/1731613034826/Put/seqid=0 2024-11-14T19:37:16,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742340_1516 (size=30955) 2024-11-14T19:37:16,861 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T19:37:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T19:37:16,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:16,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:16,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613096987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:16,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613096993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:16,999 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:17,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613097003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:17,007 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613097003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613097003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:17,008 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:17,008 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:17,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613097093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:17,222 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/efb57c24e3b24e9caec56be7ffdf757a 2024-11-14T19:37:17,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/0fbf2d87f0664868b41451d1f5c4f886 is 50, key is test_row_0/B:col10/1731613034826/Put/seqid=0 2024-11-14T19:37:17,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742341_1517 (size=12001) 2024-11-14T19:37:17,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613097303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:17,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613097613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:17,675 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/0fbf2d87f0664868b41451d1f5c4f886 2024-11-14T19:37:17,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a2c99c2139754560ada9ac6a1e15f5a6 is 50, key is test_row_0/C:col10/1731613034826/Put/seqid=0 2024-11-14T19:37:17,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742342_1518 (size=12001) 2024-11-14T19:37:17,717 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a2c99c2139754560ada9ac6a1e15f5a6 2024-11-14T19:37:17,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/efb57c24e3b24e9caec56be7ffdf757a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/efb57c24e3b24e9caec56be7ffdf757a 2024-11-14T19:37:17,730 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/efb57c24e3b24e9caec56be7ffdf757a, entries=150, sequenceid=50, filesize=30.2 K 2024-11-14T19:37:17,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/0fbf2d87f0664868b41451d1f5c4f886 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/0fbf2d87f0664868b41451d1f5c4f886 2024-11-14T19:37:17,740 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/0fbf2d87f0664868b41451d1f5c4f886, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T19:37:17,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a2c99c2139754560ada9ac6a1e15f5a6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a2c99c2139754560ada9ac6a1e15f5a6 2024-11-14T19:37:17,746 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a2c99c2139754560ada9ac6a1e15f5a6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-14T19:37:17,747 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 12ed89d0dac4353a0596c5193935d87c in 1463ms, sequenceid=50, compaction requested=false 2024-11-14T19:37:17,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:17,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:17,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-14T19:37:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-14T19:37:17,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-14T19:37:17,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9990 sec 2024-11-14T19:37:17,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.0090 sec 2024-11-14T19:37:18,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:18,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:37:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:18,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:18,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114305d060e79ce4591849c3be0790107f7_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613038123/Put/seqid=0 2024-11-14T19:37:18,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742343_1519 (size=14594) 2024-11-14T19:37:18,176 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:18,184 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114305d060e79ce4591849c3be0790107f7_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114305d060e79ce4591849c3be0790107f7_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:18,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/dec502adcb564931b521617ee7d6d2e7, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:18,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/dec502adcb564931b521617ee7d6d2e7 is 175, key is test_row_0/A:col10/1731613038123/Put/seqid=0 2024-11-14T19:37:18,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613038196, max timestamp=1731613038221Num uniques 859; 2024-11-14T19:37:18,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:18,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742344_1520 (size=39549) 2024-11-14T19:37:18,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/dec502adcb564931b521617ee7d6d2e7 2024-11-14T19:37:18,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613038196, max timestamp=1731613038221Num uniques 859; 2024-11-14T19:37:18,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613038196, max timestamp=1731613038221Num uniques 859; 2024-11-14T19:37:18,228 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:18,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/9fe3fa6c20f047d9a61aa7babc9e80a8 is 50, key is test_row_0/B:col10/1731613038123/Put/seqid=0 2024-11-14T19:37:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742345_1521 (size=12001) 2024-11-14T19:37:18,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/9fe3fa6c20f047d9a61aa7babc9e80a8 2024-11-14T19:37:18,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a77597d4b6d648e395b1aa6756b1003f is 50, key is test_row_0/C:col10/1731613038123/Put/seqid=0 2024-11-14T19:37:18,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742346_1522 (size=12001) 2024-11-14T19:37:18,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a77597d4b6d648e395b1aa6756b1003f 2024-11-14T19:37:18,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/dec502adcb564931b521617ee7d6d2e7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/dec502adcb564931b521617ee7d6d2e7 2024-11-14T19:37:18,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/dec502adcb564931b521617ee7d6d2e7, entries=200, sequenceid=60, filesize=38.6 K 2024-11-14T19:37:18,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/9fe3fa6c20f047d9a61aa7babc9e80a8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9fe3fa6c20f047d9a61aa7babc9e80a8 2024-11-14T19:37:18,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9fe3fa6c20f047d9a61aa7babc9e80a8, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T19:37:18,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a77597d4b6d648e395b1aa6756b1003f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a77597d4b6d648e395b1aa6756b1003f 2024-11-14T19:37:18,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a77597d4b6d648e395b1aa6756b1003f, entries=150, sequenceid=60, filesize=11.7 K 2024-11-14T19:37:18,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=107.34 KB/109920 for 12ed89d0dac4353a0596c5193935d87c in 197ms, sequenceid=60, compaction requested=true 2024-11-14T19:37:18,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:18,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:18,322 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:18,322 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:18,323 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:18,323 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:18,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:18,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:18,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:18,324 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:18,324 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:18,325 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:18,325 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/76250cbadf14464e83f4663dd2871b5c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/0fbf2d87f0664868b41451d1f5c4f886, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9fe3fa6c20f047d9a61aa7babc9e80a8] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=35.2 K 2024-11-14T19:37:18,325 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:18,325 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:18,325 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 76250cbadf14464e83f4663dd2871b5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731613034749 2024-11-14T19:37:18,325 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:18,325 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5c4f35696ebb42e1877d261868d2c2db, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/efb57c24e3b24e9caec56be7ffdf757a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/dec502adcb564931b521617ee7d6d2e7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=107.5 K 2024-11-14T19:37:18,325 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:18,325 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5c4f35696ebb42e1877d261868d2c2db, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/efb57c24e3b24e9caec56be7ffdf757a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/dec502adcb564931b521617ee7d6d2e7] 2024-11-14T19:37:18,326 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fbf2d87f0664868b41451d1f5c4f886, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731613034808 2024-11-14T19:37:18,327 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4f35696ebb42e1877d261868d2c2db, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731613034749 2024-11-14T19:37:18,327 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe3fa6c20f047d9a61aa7babc9e80a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731613036967 2024-11-14T19:37:18,327 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb57c24e3b24e9caec56be7ffdf757a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731613034808 2024-11-14T19:37:18,328 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting dec502adcb564931b521617ee7d6d2e7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731613036960 2024-11-14T19:37:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:18,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=114.05 KB heapSize=325.22 KB 2024-11-14T19:37:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:18,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:18,355 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:18,356 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/6888275980c3474faab48ad54631cba7 is 50, key is test_row_0/B:col10/1731613038123/Put/seqid=0 2024-11-14T19:37:18,359 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:18,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140e0100dcc56a44baa7aa8647f02b3ee2_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613038285/Put/seqid=0 2024-11-14T19:37:18,386 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149e83eededc184540bd53f239476b8dfd_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:18,389 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149e83eededc184540bd53f239476b8dfd_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:18,389 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149e83eededc184540bd53f239476b8dfd_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:18,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:18,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613098405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:18,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:18,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742347_1523 (size=12104) 2024-11-14T19:37:18,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742348_1524 (size=14594) 2024-11-14T19:37:18,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742349_1525 (size=4469) 2024-11-14T19:37:18,475 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#441 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:18,476 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/909ae297420e46e7845447e37545ce6c is 175, key is test_row_0/A:col10/1731613038123/Put/seqid=0 2024-11-14T19:37:18,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742350_1526 (size=31058) 2024-11-14T19:37:18,513 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/909ae297420e46e7845447e37545ce6c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/909ae297420e46e7845447e37545ce6c 2024-11-14T19:37:18,521 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into 909ae297420e46e7845447e37545ce6c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:18,521 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:18,521 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=13, startTime=1731613038322; duration=0sec 2024-11-14T19:37:18,521 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:18,521 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:18,521 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:18,522 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:18,522 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:18,522 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:18,522 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8397fa4eef1a474991529708f892fa0e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a2c99c2139754560ada9ac6a1e15f5a6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a77597d4b6d648e395b1aa6756b1003f] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=35.2 K 2024-11-14T19:37:18,523 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8397fa4eef1a474991529708f892fa0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731613034749 2024-11-14T19:37:18,523 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c99c2139754560ada9ac6a1e15f5a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731613034808 2024-11-14T19:37:18,523 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a77597d4b6d648e395b1aa6756b1003f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731613036967 2024-11-14T19:37:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613098523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:18,544 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#443 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:18,545 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/24d7dd906e3a472089617b6a59f4a585 is 50, key is test_row_0/C:col10/1731613038123/Put/seqid=0 2024-11-14T19:37:18,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742351_1527 (size=12104) 2024-11-14T19:37:18,592 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/24d7dd906e3a472089617b6a59f4a585 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/24d7dd906e3a472089617b6a59f4a585 2024-11-14T19:37:18,605 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 24d7dd906e3a472089617b6a59f4a585(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:18,605 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:18,605 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=13, startTime=1731613038324; duration=0sec 2024-11-14T19:37:18,605 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:18,605 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:18,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613098733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:18,835 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/6888275980c3474faab48ad54631cba7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/6888275980c3474faab48ad54631cba7 2024-11-14T19:37:18,841 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into 6888275980c3474faab48ad54631cba7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:18,841 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:18,841 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=13, startTime=1731613038323; duration=0sec 2024-11-14T19:37:18,842 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:18,842 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:18,846 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:18,849 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411140e0100dcc56a44baa7aa8647f02b3ee2_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140e0100dcc56a44baa7aa8647f02b3ee2_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:18,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/345795b8a5bd48dfaf1cef5eb5d1af5d, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:18,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/345795b8a5bd48dfaf1cef5eb5d1af5d is 175, key is test_row_0/A:col10/1731613038285/Put/seqid=0 2024-11-14T19:37:18,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742352_1528 (size=39549) 2024-11-14T19:37:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-14T19:37:18,914 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:18,914 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-14T19:37:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T19:37:18,916 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:18,917 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:18,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:19,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613099013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:19,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613099013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:19,017 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:19,018 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T19:37:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613099023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:19,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613099023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:19,026 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:19,027 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_2' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613099043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:19,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T19:37:19,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:19,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T19:37:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:19,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T19:37:19,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/345795b8a5bd48dfaf1cef5eb5d1af5d 2024-11-14T19:37:19,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/27009b92f24540559d74205f1650442f is 50, key is test_row_0/B:col10/1731613038285/Put/seqid=0 2024-11-14T19:37:19,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742353_1529 (size=12001) 2024-11-14T19:37:19,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/27009b92f24540559d74205f1650442f 2024-11-14T19:37:19,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T19:37:19,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:19,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/d5a647a9d40b4400a47cb5939d4453aa is 50, key is test_row_0/C:col10/1731613038285/Put/seqid=0 2024-11-14T19:37:19,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742354_1530 (size=12001) 2024-11-14T19:37:19,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T19:37:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T19:37:19,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:19,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613099553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:19,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T19:37:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T19:37:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:19,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/d5a647a9d40b4400a47cb5939d4453aa 2024-11-14T19:37:19,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/345795b8a5bd48dfaf1cef5eb5d1af5d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/345795b8a5bd48dfaf1cef5eb5d1af5d 2024-11-14T19:37:19,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/345795b8a5bd48dfaf1cef5eb5d1af5d, entries=200, sequenceid=94, filesize=38.6 K 2024-11-14T19:37:19,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/27009b92f24540559d74205f1650442f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/27009b92f24540559d74205f1650442f 2024-11-14T19:37:19,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/27009b92f24540559d74205f1650442f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-14T19:37:19,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/d5a647a9d40b4400a47cb5939d4453aa as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/d5a647a9d40b4400a47cb5939d4453aa 2024-11-14T19:37:19,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/d5a647a9d40b4400a47cb5939d4453aa, entries=150, sequenceid=94, filesize=11.7 K 2024-11-14T19:37:19,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~325.17 KB/332976, currentSize=67.09 KB/68700 for 12ed89d0dac4353a0596c5193935d87c in 1543ms, sequenceid=94, compaction requested=false 2024-11-14T19:37:19,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:19,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-14T19:37:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:19,997 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:37:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:20,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c8242525b3f54da5a439d83cc9c9f54a_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613038403/Put/seqid=0 2024-11-14T19:37:20,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742355_1531 (size=12154) 2024-11-14T19:37:20,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:20,050 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114c8242525b3f54da5a439d83cc9c9f54a_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c8242525b3f54da5a439d83cc9c9f54a_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/3f47325214404e8980cd6de3ade374fb, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:20,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/3f47325214404e8980cd6de3ade374fb is 175, key is test_row_0/A:col10/1731613038403/Put/seqid=0 2024-11-14T19:37:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T19:37:20,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742356_1532 (size=30955) 2024-11-14T19:37:20,461 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/3f47325214404e8980cd6de3ade374fb 2024-11-14T19:37:20,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/f2b237cc46114c0384945dde7913b6b7 is 50, key is test_row_0/B:col10/1731613038403/Put/seqid=0 2024-11-14T19:37:20,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742357_1533 (size=12001) 2024-11-14T19:37:20,543 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/f2b237cc46114c0384945dde7913b6b7 2024-11-14T19:37:20,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/3e3354d408ee43288770a1f55de7e0ac is 50, key is test_row_0/C:col10/1731613038403/Put/seqid=0 2024-11-14T19:37:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:20,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:20,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742358_1534 (size=12001) 2024-11-14T19:37:20,599 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/3e3354d408ee43288770a1f55de7e0ac 2024-11-14T19:37:20,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/3f47325214404e8980cd6de3ade374fb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/3f47325214404e8980cd6de3ade374fb 2024-11-14T19:37:20,608 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/3f47325214404e8980cd6de3ade374fb, entries=150, sequenceid=110, filesize=30.2 K 2024-11-14T19:37:20,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/f2b237cc46114c0384945dde7913b6b7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2b237cc46114c0384945dde7913b6b7 2024-11-14T19:37:20,614 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2b237cc46114c0384945dde7913b6b7, entries=150, sequenceid=110, filesize=11.7 K 2024-11-14T19:37:20,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/3e3354d408ee43288770a1f55de7e0ac as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/3e3354d408ee43288770a1f55de7e0ac 2024-11-14T19:37:20,623 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/3e3354d408ee43288770a1f55de7e0ac, entries=150, sequenceid=110, filesize=11.7 K 2024-11-14T19:37:20,625 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=60.38 KB/61830 for 12ed89d0dac4353a0596c5193935d87c in 627ms, sequenceid=110, compaction requested=true 2024-11-14T19:37:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-14T19:37:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-14T19:37:20,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:37:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:20,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:20,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-14T19:37:20,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-11-14T19:37:20,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.7180 sec 2024-11-14T19:37:20,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a58a26b2c9d94a02ae6d462a545b4915_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613040575/Put/seqid=0 2024-11-14T19:37:20,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742359_1535 (size=14594) 2024-11-14T19:37:20,676 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:20,680 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a58a26b2c9d94a02ae6d462a545b4915_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a58a26b2c9d94a02ae6d462a545b4915_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:20,681 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bc7cfb73c67f4793becb3fd0a8bbeb48, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:20,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bc7cfb73c67f4793becb3fd0a8bbeb48 is 175, key is test_row_0/A:col10/1731613040575/Put/seqid=0 2024-11-14T19:37:20,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742360_1536 (size=39549) 2024-11-14T19:37:20,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bc7cfb73c67f4793becb3fd0a8bbeb48 2024-11-14T19:37:20,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/a119cc711f6443b8b02d90231f724c55 is 50, key is test_row_0/B:col10/1731613040575/Put/seqid=0 2024-11-14T19:37:20,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742361_1537 (size=12001) 2024-11-14T19:37:20,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/a119cc711f6443b8b02d90231f724c55 2024-11-14T19:37:20,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613100743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:20,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0058ac3c2f974024888cd4fccc51695d is 50, key is test_row_0/C:col10/1731613040575/Put/seqid=0 2024-11-14T19:37:20,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742362_1538 (size=12001) 2024-11-14T19:37:20,813 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0058ac3c2f974024888cd4fccc51695d 2024-11-14T19:37:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bc7cfb73c67f4793becb3fd0a8bbeb48 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bc7cfb73c67f4793becb3fd0a8bbeb48 2024-11-14T19:37:20,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bc7cfb73c67f4793becb3fd0a8bbeb48, entries=200, sequenceid=123, filesize=38.6 K 2024-11-14T19:37:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/a119cc711f6443b8b02d90231f724c55 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a119cc711f6443b8b02d90231f724c55 2024-11-14T19:37:20,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a119cc711f6443b8b02d90231f724c55, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T19:37:20,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0058ac3c2f974024888cd4fccc51695d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0058ac3c2f974024888cd4fccc51695d 2024-11-14T19:37:20,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0058ac3c2f974024888cd4fccc51695d, entries=150, sequenceid=123, filesize=11.7 K 2024-11-14T19:37:20,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 12ed89d0dac4353a0596c5193935d87c in 218ms, sequenceid=123, compaction requested=true 2024-11-14T19:37:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:20,844 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:20,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:20,845 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:20,845 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:20,846 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:20,846 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:20,846 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:20,846 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:20,846 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:20,846 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/909ae297420e46e7845447e37545ce6c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/345795b8a5bd48dfaf1cef5eb5d1af5d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/3f47325214404e8980cd6de3ade374fb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bc7cfb73c67f4793becb3fd0a8bbeb48] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=137.8 K 2024-11-14T19:37:20,846 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/6888275980c3474faab48ad54631cba7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/27009b92f24540559d74205f1650442f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2b237cc46114c0384945dde7913b6b7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a119cc711f6443b8b02d90231f724c55] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=47.0 K 2024-11-14T19:37:20,846 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:20,846 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/909ae297420e46e7845447e37545ce6c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/345795b8a5bd48dfaf1cef5eb5d1af5d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/3f47325214404e8980cd6de3ade374fb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bc7cfb73c67f4793becb3fd0a8bbeb48] 2024-11-14T19:37:20,847 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6888275980c3474faab48ad54631cba7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731613036967 2024-11-14T19:37:20,847 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 909ae297420e46e7845447e37545ce6c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731613036967 2024-11-14T19:37:20,847 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27009b92f24540559d74205f1650442f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731613038285 2024-11-14T19:37:20,847 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 345795b8a5bd48dfaf1cef5eb5d1af5d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731613038285 2024-11-14T19:37:20,847 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b237cc46114c0384945dde7913b6b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731613038373 2024-11-14T19:37:20,847 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f47325214404e8980cd6de3ade374fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731613038373 2024-11-14T19:37:20,848 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting a119cc711f6443b8b02d90231f724c55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731613040575 2024-11-14T19:37:20,848 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7cfb73c67f4793becb3fd0a8bbeb48, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731613040575 2024-11-14T19:37:20,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:20,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:37:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:20,869 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#452 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:20,870 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/2a7c44d70f7b49c1acea041d30de40e1 is 50, key is test_row_0/B:col10/1731613040575/Put/seqid=0 2024-11-14T19:37:20,879 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:20,895 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114397ffcf0cad249b8bcc5a339eb675389_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:20,897 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114397ffcf0cad249b8bcc5a339eb675389_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:20,898 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114397ffcf0cad249b8bcc5a339eb675389_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:20,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145005c5fc648c41aa84ed805bae4f65f7_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613040700/Put/seqid=0 2024-11-14T19:37:20,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742363_1539 (size=12241) 2024-11-14T19:37:20,912 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/2a7c44d70f7b49c1acea041d30de40e1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2a7c44d70f7b49c1acea041d30de40e1 2024-11-14T19:37:20,922 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into 2a7c44d70f7b49c1acea041d30de40e1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:20,923 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:20,923 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=12, startTime=1731613040844; duration=0sec 2024-11-14T19:37:20,923 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:20,923 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:20,923 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:20,924 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:20,924 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:20,925 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:20,925 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/24d7dd906e3a472089617b6a59f4a585, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/d5a647a9d40b4400a47cb5939d4453aa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/3e3354d408ee43288770a1f55de7e0ac, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0058ac3c2f974024888cd4fccc51695d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=47.0 K 2024-11-14T19:37:20,925 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d7dd906e3a472089617b6a59f4a585, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731613036967 2024-11-14T19:37:20,926 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5a647a9d40b4400a47cb5939d4453aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731613038285 2024-11-14T19:37:20,927 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3354d408ee43288770a1f55de7e0ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731613038373 2024-11-14T19:37:20,928 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0058ac3c2f974024888cd4fccc51695d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731613040575 2024-11-14T19:37:20,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742364_1540 (size=4469) 2024-11-14T19:37:20,938 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#453 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:20,939 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/d2a08e5362624022a3ebe931ff23c8e2 is 175, key is test_row_0/A:col10/1731613040575/Put/seqid=0 2024-11-14T19:37:20,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613100951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:20,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:20,965 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:20,965 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/939f488e5b5c41c1b618f293dc993004 is 50, key is test_row_0/C:col10/1731613040575/Put/seqid=0 2024-11-14T19:37:20,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742365_1541 (size=14794) 2024-11-14T19:37:21,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742366_1542 (size=31195) 2024-11-14T19:37:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-14T19:37:21,064 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:21,064 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:21,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613101064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:21,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:21,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-14T19:37:21,066 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T19:37:21,067 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:21,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:21,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742367_1543 (size=12241) 2024-11-14T19:37:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T19:37:21,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613101273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:21,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:21,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,384 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T19:37:21,387 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145005c5fc648c41aa84ed805bae4f65f7_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145005c5fc648c41aa84ed805bae4f65f7_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:21,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0ac0de1775764403bec8431b27e8fc53, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:21,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0ac0de1775764403bec8431b27e8fc53 is 175, key is test_row_0/A:col10/1731613040700/Put/seqid=0 2024-11-14T19:37:21,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742368_1544 (size=39749) 2024-11-14T19:37:21,407 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0ac0de1775764403bec8431b27e8fc53 2024-11-14T19:37:21,441 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/d2a08e5362624022a3ebe931ff23c8e2 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/d2a08e5362624022a3ebe931ff23c8e2 2024-11-14T19:37:21,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/eaa4aea17fc0449f859518a45aa1b5ac is 50, key is test_row_0/B:col10/1731613040700/Put/seqid=0 2024-11-14T19:37:21,454 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into d2a08e5362624022a3ebe931ff23c8e2(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:21,458 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:21,458 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=12, startTime=1731613040844; duration=0sec 2024-11-14T19:37:21,458 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:21,458 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:21,487 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/939f488e5b5c41c1b618f293dc993004 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/939f488e5b5c41c1b618f293dc993004 2024-11-14T19:37:21,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742369_1545 (size=12151) 2024-11-14T19:37:21,497 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 939f488e5b5c41c1b618f293dc993004(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:21,497 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:21,497 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=12, startTime=1731613040844; duration=0sec 2024-11-14T19:37:21,497 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:21,497 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:21,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:21,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:21,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613101583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:21,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T19:37:21,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:21,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/eaa4aea17fc0449f859518a45aa1b5ac 2024-11-14T19:37:21,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/826b1aa1da4448c0ae60a3e3382b7d02 is 50, key is test_row_0/C:col10/1731613040700/Put/seqid=0 2024-11-14T19:37:21,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742370_1546 (size=12151) 2024-11-14T19:37:21,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:21,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:22,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613102093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:22,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:22,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:22,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:22,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:22,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:22,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T19:37:22,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:22,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:22,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:22,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:22,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:22,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:22,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/826b1aa1da4448c0ae60a3e3382b7d02 2024-11-14T19:37:22,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0ac0de1775764403bec8431b27e8fc53 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0ac0de1775764403bec8431b27e8fc53 2024-11-14T19:37:22,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0ac0de1775764403bec8431b27e8fc53, entries=200, sequenceid=143, filesize=38.8 K 2024-11-14T19:37:22,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/eaa4aea17fc0449f859518a45aa1b5ac as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/eaa4aea17fc0449f859518a45aa1b5ac 2024-11-14T19:37:22,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/eaa4aea17fc0449f859518a45aa1b5ac, entries=150, sequenceid=143, filesize=11.9 K 2024-11-14T19:37:22,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/826b1aa1da4448c0ae60a3e3382b7d02 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/826b1aa1da4448c0ae60a3e3382b7d02 2024-11-14T19:37:22,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/826b1aa1da4448c0ae60a3e3382b7d02, entries=150, sequenceid=143, filesize=11.9 K 2024-11-14T19:37:22,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 12ed89d0dac4353a0596c5193935d87c in 1494ms, sequenceid=143, compaction requested=false 2024-11-14T19:37:22,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:22,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-14T19:37:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:22,454 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:37:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:22,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:22,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114632e390f22b443c7a5771ce5ee88a904_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613040927/Put/seqid=0 2024-11-14T19:37:22,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742371_1547 (size=12304) 2024-11-14T19:37:22,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,533 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114632e390f22b443c7a5771ce5ee88a904_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114632e390f22b443c7a5771ce5ee88a904_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:22,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fa3d5d795e7e4151817cc2ff628f1ec8, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:22,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fa3d5d795e7e4151817cc2ff628f1ec8 is 175, key is test_row_0/A:col10/1731613040927/Put/seqid=0 2024-11-14T19:37:22,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742372_1548 (size=31105) 2024-11-14T19:37:22,562 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fa3d5d795e7e4151817cc2ff628f1ec8 2024-11-14T19:37:22,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/5cd840a0677440959c2817e74170bdbd is 50, key is test_row_0/B:col10/1731613040927/Put/seqid=0 2024-11-14T19:37:22,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742373_1549 (size=12151) 2024-11-14T19:37:22,611 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/5cd840a0677440959c2817e74170bdbd 2024-11-14T19:37:22,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/ee30f89f986c4f3c8b9cecf7e94bfc6f is 50, key is test_row_0/C:col10/1731613040927/Put/seqid=0 2024-11-14T19:37:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742374_1550 (size=12151) 2024-11-14T19:37:22,649 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/ee30f89f986c4f3c8b9cecf7e94bfc6f 2024-11-14T19:37:22,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fa3d5d795e7e4151817cc2ff628f1ec8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fa3d5d795e7e4151817cc2ff628f1ec8 2024-11-14T19:37:22,667 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fa3d5d795e7e4151817cc2ff628f1ec8, entries=150, sequenceid=158, filesize=30.4 K 2024-11-14T19:37:22,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/5cd840a0677440959c2817e74170bdbd as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/5cd840a0677440959c2817e74170bdbd 2024-11-14T19:37:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,671 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/5cd840a0677440959c2817e74170bdbd, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T19:37:22,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/ee30f89f986c4f3c8b9cecf7e94bfc6f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/ee30f89f986c4f3c8b9cecf7e94bfc6f 2024-11-14T19:37:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,678 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/ee30f89f986c4f3c8b9cecf7e94bfc6f, entries=150, sequenceid=158, filesize=11.9 K 2024-11-14T19:37:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,679 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 12ed89d0dac4353a0596c5193935d87c in 224ms, sequenceid=158, compaction requested=true 2024-11-14T19:37:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-14T19:37:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-14T19:37:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-14T19:37:22,682 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-14T19:37:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.6190 sec 2024-11-14T19:37:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:37:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:23,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:23,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:23,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:23,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:23,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:23,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114669c95344e2d46769e0a334acd642c38_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613043076/Put/seqid=0 2024-11-14T19:37:23,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731613043117, max timestamp=1731613043135Num uniques 859; 2024-11-14T19:37:23,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:37:23,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731613043117, max timestamp=1731613043135Num uniques 859; 2024-11-14T19:37:23,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:37:23,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731613043117, max timestamp=1731613043135Num uniques 859; 2024-11-14T19:37:23,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-14T19:37:23,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742375_1551 (size=22268) 2024-11-14T19:37:23,154 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:23,164 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114669c95344e2d46769e0a334acd642c38_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114669c95344e2d46769e0a334acd642c38_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:23,165 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5b75ecb959864d0f9c365038647dd9ba, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:23,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5b75ecb959864d0f9c365038647dd9ba is 175, key is test_row_0/A:col10/1731613043076/Put/seqid=0 2024-11-14T19:37:23,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613103183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613103192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613103194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613103197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613103199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-14T19:37:23,214 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:23,214 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-14T19:37:23,216 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:23,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742376_1552 (size=65673) 2024-11-14T19:37:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T19:37:23,217 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5b75ecb959864d0f9c365038647dd9ba 2024-11-14T19:37:23,218 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:23,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:23,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/38921ba854b242beaa09f937dc119c5d is 50, key is test_row_0/B:col10/1731613043076/Put/seqid=0 2024-11-14T19:37:23,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742377_1553 (size=12151) 2024-11-14T19:37:23,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613103303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613103313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613103314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613103314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613103314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T19:37:23,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T19:37:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613103513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T19:37:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613103523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613103523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613103523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613103523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T19:37:23,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/38921ba854b242beaa09f937dc119c5d 2024-11-14T19:37:23,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T19:37:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/70c5e91d7ade472396e08745c73d541e is 50, key is test_row_0/C:col10/1731613043076/Put/seqid=0 2024-11-14T19:37:23,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742378_1554 (size=12151) 2024-11-14T19:37:23,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613103823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T19:37:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613103834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613103834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T19:37:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613103844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613103844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:23,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:23,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T19:37:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:23,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:24,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/70c5e91d7ade472396e08745c73d541e 2024-11-14T19:37:24,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/5b75ecb959864d0f9c365038647dd9ba as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5b75ecb959864d0f9c365038647dd9ba 2024-11-14T19:37:24,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5b75ecb959864d0f9c365038647dd9ba, entries=350, sequenceid=168, filesize=64.1 K 2024-11-14T19:37:24,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/38921ba854b242beaa09f937dc119c5d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/38921ba854b242beaa09f937dc119c5d 2024-11-14T19:37:24,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/38921ba854b242beaa09f937dc119c5d, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T19:37:24,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/70c5e91d7ade472396e08745c73d541e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/70c5e91d7ade472396e08745c73d541e 2024-11-14T19:37:24,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/70c5e91d7ade472396e08745c73d541e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-14T19:37:24,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 12ed89d0dac4353a0596c5193935d87c in 1051ms, sequenceid=168, compaction requested=true 2024-11-14T19:37:24,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:24,132 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:24,133 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:24,133 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:24,133 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:24,134 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/d2a08e5362624022a3ebe931ff23c8e2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0ac0de1775764403bec8431b27e8fc53, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fa3d5d795e7e4151817cc2ff628f1ec8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5b75ecb959864d0f9c365038647dd9ba] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=163.8 K 2024-11-14T19:37:24,134 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:24,134 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/d2a08e5362624022a3ebe931ff23c8e2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0ac0de1775764403bec8431b27e8fc53, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fa3d5d795e7e4151817cc2ff628f1ec8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5b75ecb959864d0f9c365038647dd9ba] 2024-11-14T19:37:24,134 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2a08e5362624022a3ebe931ff23c8e2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731613040575 2024-11-14T19:37:24,134 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac0de1775764403bec8431b27e8fc53, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731613040700 2024-11-14T19:37:24,135 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa3d5d795e7e4151817cc2ff628f1ec8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731613040927 2024-11-14T19:37:24,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:24,135 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b75ecb959864d0f9c365038647dd9ba, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731613043047 2024-11-14T19:37:24,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:24,138 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:24,139 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:24,139 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:24,139 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:24,140 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2a7c44d70f7b49c1acea041d30de40e1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/eaa4aea17fc0449f859518a45aa1b5ac, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/5cd840a0677440959c2817e74170bdbd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/38921ba854b242beaa09f937dc119c5d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=47.6 K 2024-11-14T19:37:24,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-14T19:37:24,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:24,142 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-14T19:37:24,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:24,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:24,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:24,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:24,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:24,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:24,147 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a7c44d70f7b49c1acea041d30de40e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731613040575 2024-11-14T19:37:24,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:24,147 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa4aea17fc0449f859518a45aa1b5ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731613040700 2024-11-14T19:37:24,148 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd840a0677440959c2817e74170bdbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731613040927 2024-11-14T19:37:24,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:24,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:24,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:24,151 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 38921ba854b242beaa09f937dc119c5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731613043066 2024-11-14T19:37:24,153 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:24,173 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411145154fbe1b9614b87866aeaf4a15f90df_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:24,176 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411145154fbe1b9614b87866aeaf4a15f90df_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:24,176 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411145154fbe1b9614b87866aeaf4a15f90df_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:24,190 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:24,190 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/4efb6d3db1fd43218cf63c5df7c98cf4 is 50, key is test_row_0/B:col10/1731613043076/Put/seqid=0 2024-11-14T19:37:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144823755deb7b44fd8b0646e52ed9e3ae_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613043194/Put/seqid=0 2024-11-14T19:37:24,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742380_1556 (size=12527) 2024-11-14T19:37:24,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742379_1555 (size=4469) 2024-11-14T19:37:24,268 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#464 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:24,268 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/21d70ce5a3a440799d722697ab01d02f is 175, key is test_row_0/A:col10/1731613043076/Put/seqid=0 2024-11-14T19:37:24,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742381_1557 (size=12304) 2024-11-14T19:37:24,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,297 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411144823755deb7b44fd8b0646e52ed9e3ae_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144823755deb7b44fd8b0646e52ed9e3ae_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fbcaa84735344c95b972adf8e6203959, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:24,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fbcaa84735344c95b972adf8e6203959 is 175, key is test_row_0/A:col10/1731613043194/Put/seqid=0 2024-11-14T19:37:24,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742382_1558 (size=31481) 2024-11-14T19:37:24,330 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/21d70ce5a3a440799d722697ab01d02f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/21d70ce5a3a440799d722697ab01d02f 2024-11-14T19:37:24,335 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into 21d70ce5a3a440799d722697ab01d02f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:24,335 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:24,335 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=12, startTime=1731613044132; duration=0sec 2024-11-14T19:37:24,335 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:24,335 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:24,335 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T19:37:24,336 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-14T19:37:24,336 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:24,336 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:24,336 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/939f488e5b5c41c1b618f293dc993004, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/826b1aa1da4448c0ae60a3e3382b7d02, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/ee30f89f986c4f3c8b9cecf7e94bfc6f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/70c5e91d7ade472396e08745c73d541e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=47.6 K 2024-11-14T19:37:24,336 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939f488e5b5c41c1b618f293dc993004, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731613040575 2024-11-14T19:37:24,337 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826b1aa1da4448c0ae60a3e3382b7d02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1731613040700 2024-11-14T19:37:24,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:24,337 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee30f89f986c4f3c8b9cecf7e94bfc6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731613040927 2024-11-14T19:37:24,337 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c5e91d7ade472396e08745c73d541e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731613043066 2024-11-14T19:37:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742383_1559 (size=31105) 2024-11-14T19:37:24,352 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fbcaa84735344c95b972adf8e6203959 2024-11-14T19:37:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T19:37:24,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613104354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613104356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613104358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613104360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613104363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,373 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:24,374 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/2d0740c329714d57bdddaa5d61105e06 is 50, key is test_row_0/C:col10/1731613043076/Put/seqid=0 2024-11-14T19:37:24,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/a360214223044f5fbf41390b64f3af3f is 50, key is test_row_0/B:col10/1731613043194/Put/seqid=0 2024-11-14T19:37:24,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742384_1560 (size=12527) 2024-11-14T19:37:24,419 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/2d0740c329714d57bdddaa5d61105e06 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/2d0740c329714d57bdddaa5d61105e06 2024-11-14T19:37:24,425 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 2d0740c329714d57bdddaa5d61105e06(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:24,425 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:24,425 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=12, startTime=1731613044149; duration=0sec 2024-11-14T19:37:24,425 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:24,425 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:24,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742385_1561 (size=12151) 2024-11-14T19:37:24,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613104473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613104473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613104473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,660 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/4efb6d3db1fd43218cf63c5df7c98cf4 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4efb6d3db1fd43218cf63c5df7c98cf4 2024-11-14T19:37:24,665 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into 4efb6d3db1fd43218cf63c5df7c98cf4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:24,665 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:24,665 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=12, startTime=1731613044138; duration=0sec 2024-11-14T19:37:24,665 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:24,665 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:24,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613104684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613104684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613104684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:24,849 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/a360214223044f5fbf41390b64f3af3f 2024-11-14T19:37:24,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/95216910f7114619b45733ee6cf89bde is 50, key is test_row_0/C:col10/1731613043194/Put/seqid=0 2024-11-14T19:37:24,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742386_1562 (size=12151) 2024-11-14T19:37:24,917 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/95216910f7114619b45733ee6cf89bde 2024-11-14T19:37:24,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fbcaa84735344c95b972adf8e6203959 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fbcaa84735344c95b972adf8e6203959 2024-11-14T19:37:24,924 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fbcaa84735344c95b972adf8e6203959, entries=150, sequenceid=204, filesize=30.4 K 2024-11-14T19:37:24,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/a360214223044f5fbf41390b64f3af3f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a360214223044f5fbf41390b64f3af3f 2024-11-14T19:37:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,929 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a360214223044f5fbf41390b64f3af3f, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T19:37:24,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/95216910f7114619b45733ee6cf89bde as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/95216910f7114619b45733ee6cf89bde 2024-11-14T19:37:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,934 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/95216910f7114619b45733ee6cf89bde, entries=150, sequenceid=204, filesize=11.9 K 2024-11-14T19:37:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,935 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 12ed89d0dac4353a0596c5193935d87c in 794ms, sequenceid=204, compaction requested=false 2024-11-14T19:37:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-14T19:37:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-14T19:37:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-14T19:37:24,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-14T19:37:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.7240 sec 2024-11-14T19:37:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:24,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:37:24,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:24,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:24,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:24,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:24,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:24,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:25,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143def81e60ad14efea244b4f8df6d2757_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613044355/Put/seqid=0 2024-11-14T19:37:25,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613045038, max timestamp=1731613045046Num uniques 859; 2024-11-14T19:37:25,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:25,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613045038, max timestamp=1731613045046Num uniques 859; 2024-11-14T19:37:25,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:25,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613045038, max timestamp=1731613045046Num uniques 859; 2024-11-14T19:37:25,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:25,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742387_1563 (size=12304) 2024-11-14T19:37:25,063 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:25,067 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411143def81e60ad14efea244b4f8df6d2757_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143def81e60ad14efea244b4f8df6d2757_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:25,068 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/e12473c3081b48eb976bbe15e2369567, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:25,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/e12473c3081b48eb976bbe15e2369567 is 175, key is test_row_0/A:col10/1731613044355/Put/seqid=0 2024-11-14T19:37:25,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742388_1564 (size=31105) 2024-11-14T19:37:25,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731613045078, max timestamp=1731613045101Num uniques 1018; 2024-11-14T19:37:25,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731613045074, max timestamp=1731613045101Num uniques 1018; 2024-11-14T19:37:25,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-14T19:37:25,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-14T19:37:25,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731613045078, max timestamp=1731613045101Num uniques 1018; 2024-11-14T19:37:25,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7872, suffix heap size=133840,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=172 2024-11-14T19:37:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613105172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613105172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613105174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613105283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613105283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613105293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-14T19:37:25,363 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:25,364 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:25,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-14T19:37:25,365 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:25,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T19:37:25,366 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:25,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:25,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613105373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,378 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613105383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,389 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T19:37:25,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613105493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613105493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/e12473c3081b48eb976bbe15e2369567 2024-11-14T19:37:25,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613105503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/2453875028304b80bd93a93be064c76f is 50, key is test_row_0/B:col10/1731613044355/Put/seqid=0 2024-11-14T19:37:25,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T19:37:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742389_1565 (size=12151) 2024-11-14T19:37:25,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/2453875028304b80bd93a93be064c76f 2024-11-14T19:37:25,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/f5682dfe38af4943a08e0ba20aad631e is 50, key is test_row_0/C:col10/1731613044355/Put/seqid=0 2024-11-14T19:37:25,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742390_1566 (size=12151) 2024-11-14T19:37:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T19:37:25,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T19:37:25,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:25,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613105803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613105803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613105813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:25,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:25,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T19:37:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T19:37:25,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T19:37:25,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:25,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:26,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/f5682dfe38af4943a08e0ba20aad631e 2024-11-14T19:37:26,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/e12473c3081b48eb976bbe15e2369567 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/e12473c3081b48eb976bbe15e2369567 2024-11-14T19:37:26,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/e12473c3081b48eb976bbe15e2369567, entries=150, sequenceid=217, filesize=30.4 K 2024-11-14T19:37:26,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/2453875028304b80bd93a93be064c76f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2453875028304b80bd93a93be064c76f 2024-11-14T19:37:26,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2453875028304b80bd93a93be064c76f, entries=150, sequenceid=217, filesize=11.9 K 2024-11-14T19:37:26,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/f5682dfe38af4943a08e0ba20aad631e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/f5682dfe38af4943a08e0ba20aad631e 2024-11-14T19:37:26,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/f5682dfe38af4943a08e0ba20aad631e, entries=150, sequenceid=217, filesize=11.9 K 2024-11-14T19:37:26,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.36 KB/137584 for 12ed89d0dac4353a0596c5193935d87c in 1044ms, sequenceid=217, compaction requested=true 2024-11-14T19:37:26,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:26,040 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:26,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:26,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:26,041 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:26,041 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:26,042 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:26,042 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:26,042 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/21d70ce5a3a440799d722697ab01d02f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fbcaa84735344c95b972adf8e6203959, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/e12473c3081b48eb976bbe15e2369567] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=91.5 K 2024-11-14T19:37:26,042 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:26,042 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/21d70ce5a3a440799d722697ab01d02f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fbcaa84735344c95b972adf8e6203959, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/e12473c3081b48eb976bbe15e2369567] 2024-11-14T19:37:26,042 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21d70ce5a3a440799d722697ab01d02f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731613043066 2024-11-14T19:37:26,043 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:26,043 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:26,043 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:26,043 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4efb6d3db1fd43218cf63c5df7c98cf4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a360214223044f5fbf41390b64f3af3f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2453875028304b80bd93a93be064c76f] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.0 K 2024-11-14T19:37:26,043 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbcaa84735344c95b972adf8e6203959, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731613043192 2024-11-14T19:37:26,043 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4efb6d3db1fd43218cf63c5df7c98cf4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731613043066 2024-11-14T19:37:26,045 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting e12473c3081b48eb976bbe15e2369567, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731613044355 2024-11-14T19:37:26,045 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a360214223044f5fbf41390b64f3af3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731613043192 2024-11-14T19:37:26,046 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 2453875028304b80bd93a93be064c76f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731613044355 2024-11-14T19:37:26,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:26,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:26,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:26,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:26,067 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:26,080 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:26,081 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/c9a1babbcd0f4c3483bb9b69c56ac999 is 50, key is test_row_0/B:col10/1731613044355/Put/seqid=0 2024-11-14T19:37:26,095 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411142c7440ee8d514dbca4cf2d866edb4e63_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:26,097 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411142c7440ee8d514dbca4cf2d866edb4e63_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:26,098 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411142c7440ee8d514dbca4cf2d866edb4e63_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:26,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742391_1567 (size=12629) 2024-11-14T19:37:26,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742392_1568 (size=4469) 2024-11-14T19:37:26,143 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#473 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:26,143 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fe5b4e1aa9044271a84d47f5765f7cbc is 175, key is test_row_0/A:col10/1731613044355/Put/seqid=0 2024-11-14T19:37:26,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-14T19:37:26,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:26,150 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=134.36 KB heapSize=379.86 KB 2024-11-14T19:37:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:26,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:26,171 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/c9a1babbcd0f4c3483bb9b69c56ac999 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c9a1babbcd0f4c3483bb9b69c56ac999 2024-11-14T19:37:26,176 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into c9a1babbcd0f4c3483bb9b69c56ac999(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:26,176 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:26,176 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=13, startTime=1731613046041; duration=0sec 2024-11-14T19:37:26,177 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:26,177 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:26,177 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:26,178 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:26,178 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:26,178 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:26,178 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/2d0740c329714d57bdddaa5d61105e06, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/95216910f7114619b45733ee6cf89bde, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/f5682dfe38af4943a08e0ba20aad631e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.0 K 2024-11-14T19:37:26,179 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d0740c329714d57bdddaa5d61105e06, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731613043066 2024-11-14T19:37:26,180 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 95216910f7114619b45733ee6cf89bde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731613043192 2024-11-14T19:37:26,181 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f5682dfe38af4943a08e0ba20aad631e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731613044355 2024-11-14T19:37:26,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742393_1569 (size=31583) 2024-11-14T19:37:26,199 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/fe5b4e1aa9044271a84d47f5765f7cbc as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fe5b4e1aa9044271a84d47f5765f7cbc 2024-11-14T19:37:26,207 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:26,208 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/4bd05868c8304a4e869febed74595c87 is 50, key is test_row_0/C:col10/1731613044355/Put/seqid=0 2024-11-14T19:37:26,211 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into fe5b4e1aa9044271a84d47f5765f7cbc(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:26,211 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:26,211 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=13, startTime=1731613046040; duration=0sec 2024-11-14T19:37:26,211 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:26,211 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:26,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114647947190f6b4a46ba1a13f323bca286_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613045149/Put/seqid=0 2024-11-14T19:37:26,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742394_1570 (size=12629) 2024-11-14T19:37:26,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742395_1571 (size=12454) 2024-11-14T19:37:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:26,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:26,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613106336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613106341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613106341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613106453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613106453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613106453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,457 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T19:37:26,631 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/4bd05868c8304a4e869febed74595c87 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/4bd05868c8304a4e869febed74595c87 2024-11-14T19:37:26,640 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 4bd05868c8304a4e869febed74595c87(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:26,640 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:26,640 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=13, startTime=1731613046052; duration=0sec 2024-11-14T19:37:26,640 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:26,640 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:26,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613106663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613106663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613106664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:26,686 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114647947190f6b4a46ba1a13f323bca286_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114647947190f6b4a46ba1a13f323bca286_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/327282b262094cf88399c42db4ccdc0a, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/327282b262094cf88399c42db4ccdc0a is 175, key is test_row_0/A:col10/1731613045149/Put/seqid=0 2024-11-14T19:37:26,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742396_1572 (size=31255) 2024-11-14T19:37:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613106973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613106973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:26,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613106983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,132 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/327282b262094cf88399c42db4ccdc0a 2024-11-14T19:37:27,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/1545d867a77a4335b0d71c86e12aabbf is 50, key is test_row_0/B:col10/1731613045149/Put/seqid=0 2024-11-14T19:37:27,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742397_1573 (size=12301) 2024-11-14T19:37:27,210 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/1545d867a77a4335b0d71c86e12aabbf 2024-11-14T19:37:27,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0e89416bde5b49aabf377d5d19d83012 is 50, key is test_row_0/C:col10/1731613045149/Put/seqid=0 2024-11-14T19:37:27,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742398_1574 (size=12301) 2024-11-14T19:37:27,248 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.67 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0e89416bde5b49aabf377d5d19d83012 2024-11-14T19:37:27,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/327282b262094cf88399c42db4ccdc0a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/327282b262094cf88399c42db4ccdc0a 2024-11-14T19:37:27,259 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/327282b262094cf88399c42db4ccdc0a, entries=150, sequenceid=270, filesize=30.5 K 2024-11-14T19:37:27,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/1545d867a77a4335b0d71c86e12aabbf as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/1545d867a77a4335b0d71c86e12aabbf 2024-11-14T19:37:27,266 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/1545d867a77a4335b0d71c86e12aabbf, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T19:37:27,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0e89416bde5b49aabf377d5d19d83012 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0e89416bde5b49aabf377d5d19d83012 2024-11-14T19:37:27,273 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0e89416bde5b49aabf377d5d19d83012, entries=150, sequenceid=270, filesize=12.0 K 2024-11-14T19:37:27,275 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.36 KB/137584, heapSize ~379.81 KB/388928, currentSize=53.67 KB/54960 for 12ed89d0dac4353a0596c5193935d87c in 1126ms, sequenceid=270, compaction requested=false 2024-11-14T19:37:27,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:27,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:27,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-14T19:37:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-14T19:37:27,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-14T19:37:27,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9110 sec 2024-11-14T19:37:27,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9170 sec 2024-11-14T19:37:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:27,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-14T19:37:27,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:27,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:27,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148bb38b27db16470bab07bfef8d1bfca3_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613046333/Put/seqid=0 2024-11-14T19:37:27,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742399_1575 (size=14994) 2024-11-14T19:37:27,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613107468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613107470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613107483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613107483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613107493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-14T19:37:27,523 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:27,523 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-14T19:37:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T19:37:27,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:27,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:27,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:27,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613107583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613107583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T19:37:27,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:27,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:27,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:27,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:27,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613107792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613107793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:27,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:27,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,841 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T19:37:27,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411148bb38b27db16470bab07bfef8d1bfca3_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148bb38b27db16470bab07bfef8d1bfca3_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:27,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/8e5a8594203046faa7caa44e6bb2f3a1, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:27,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/8e5a8594203046faa7caa44e6bb2f3a1 is 175, key is test_row_0/A:col10/1731613046333/Put/seqid=0 2024-11-14T19:37:27,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742400_1576 (size=39949) 2024-11-14T19:37:27,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:27,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:27,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:27,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:27,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613108103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613108103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,108 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:28,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:28,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:28,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T19:37:28,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/8e5a8594203046faa7caa44e6bb2f3a1 2024-11-14T19:37:28,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ddf8a3a72eb34395acb7387ad79fc164 is 50, key is test_row_0/B:col10/1731613046333/Put/seqid=0 2024-11-14T19:37:28,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:28,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742401_1577 (size=12301) 2024-11-14T19:37:28,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613108493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:28,498 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:28,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613108503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:28,504 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:28,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613108513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:28,519 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:28,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613108613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:28,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613108613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T19:37:28,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ddf8a3a72eb34395acb7387ad79fc164 2024-11-14T19:37:28,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/8737dd7859c246c99080d521a69c0bc7 is 50, key is test_row_0/C:col10/1731613046333/Put/seqid=0 2024-11-14T19:37:28,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742402_1578 (size=12301) 2024-11-14T19:37:28,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/8737dd7859c246c99080d521a69c0bc7 2024-11-14T19:37:28,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/8e5a8594203046faa7caa44e6bb2f3a1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8e5a8594203046faa7caa44e6bb2f3a1 2024-11-14T19:37:28,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8e5a8594203046faa7caa44e6bb2f3a1, entries=200, sequenceid=285, filesize=39.0 K 2024-11-14T19:37:28,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ddf8a3a72eb34395acb7387ad79fc164 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ddf8a3a72eb34395acb7387ad79fc164 2024-11-14T19:37:28,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:28,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:28,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ddf8a3a72eb34395acb7387ad79fc164, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T19:37:28,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/8737dd7859c246c99080d521a69c0bc7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8737dd7859c246c99080d521a69c0bc7 2024-11-14T19:37:28,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8737dd7859c246c99080d521a69c0bc7, entries=150, sequenceid=285, filesize=12.0 K 2024-11-14T19:37:28,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 12ed89d0dac4353a0596c5193935d87c in 1368ms, sequenceid=285, compaction requested=true 2024-11-14T19:37:28,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:28,784 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:28,784 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:28,785 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:28,785 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:28,785 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,785 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c9a1babbcd0f4c3483bb9b69c56ac999, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/1545d867a77a4335b0d71c86e12aabbf, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ddf8a3a72eb34395acb7387ad79fc164] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.4 K 2024-11-14T19:37:28,786 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c9a1babbcd0f4c3483bb9b69c56ac999, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731613044355 2024-11-14T19:37:28,787 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1545d867a77a4335b0d71c86e12aabbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731613045149 2024-11-14T19:37:28,787 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:28,787 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:28,787 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf8a3a72eb34395acb7387ad79fc164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731613046333 2024-11-14T19:37:28,787 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,787 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fe5b4e1aa9044271a84d47f5765f7cbc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/327282b262094cf88399c42db4ccdc0a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8e5a8594203046faa7caa44e6bb2f3a1] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=100.4 K 2024-11-14T19:37:28,787 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,787 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fe5b4e1aa9044271a84d47f5765f7cbc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/327282b262094cf88399c42db4ccdc0a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8e5a8594203046faa7caa44e6bb2f3a1] 2024-11-14T19:37:28,789 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe5b4e1aa9044271a84d47f5765f7cbc, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731613044355 2024-11-14T19:37:28,791 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 327282b262094cf88399c42db4ccdc0a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731613045149 2024-11-14T19:37:28,791 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e5a8594203046faa7caa44e6bb2f3a1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731613046323 2024-11-14T19:37:28,817 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:28,826 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a23b1626f86a420a9cc61dddb7283539_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:28,827 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:28,828 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/bc686062299a4c82aadbb082e4e8dedf is 50, key is test_row_0/B:col10/1731613046333/Put/seqid=0 2024-11-14T19:37:28,828 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a23b1626f86a420a9cc61dddb7283539_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:28,829 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a23b1626f86a420a9cc61dddb7283539_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:28,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742403_1579 (size=4469) 2024-11-14T19:37:28,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742404_1580 (size=12881) 2024-11-14T19:37:28,897 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/bc686062299a4c82aadbb082e4e8dedf as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bc686062299a4c82aadbb082e4e8dedf 2024-11-14T19:37:28,905 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into bc686062299a4c82aadbb082e4e8dedf(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:28,906 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:28,906 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=13, startTime=1731613048784; duration=0sec 2024-11-14T19:37:28,906 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:28,906 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:28,906 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:28,907 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:28,907 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:28,907 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,907 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/4bd05868c8304a4e869febed74595c87, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0e89416bde5b49aabf377d5d19d83012, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8737dd7859c246c99080d521a69c0bc7] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.4 K 2024-11-14T19:37:28,908 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd05868c8304a4e869febed74595c87, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731613044355 2024-11-14T19:37:28,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-14T19:37:28,908 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e89416bde5b49aabf377d5d19d83012, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731613045149 2024-11-14T19:37:28,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:28,908 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-14T19:37:28,908 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 8737dd7859c246c99080d521a69c0bc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731613046333 2024-11-14T19:37:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:28,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:28,926 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:28,927 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/84bce474457449f8bb49589f02c1fc9a is 50, key is test_row_0/C:col10/1731613046333/Put/seqid=0 2024-11-14T19:37:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b25b334f8a8e422db833a7c46a671f5c_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613047468/Put/seqid=0 2024-11-14T19:37:28,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742405_1581 (size=12881) 2024-11-14T19:37:28,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742406_1582 (size=12454) 2024-11-14T19:37:28,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:28,995 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114b25b334f8a8e422db833a7c46a671f5c_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b25b334f8a8e422db833a7c46a671f5c_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7b4c055f949d49578a79ae148abaf45a, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7b4c055f949d49578a79ae148abaf45a is 175, key is test_row_0/A:col10/1731613047468/Put/seqid=0 2024-11-14T19:37:29,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742407_1583 (size=31255) 2024-11-14T19:37:29,006 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7b4c055f949d49578a79ae148abaf45a 2024-11-14T19:37:29,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/4abf523e3814402790af246f0266994e is 50, key is test_row_0/B:col10/1731613047468/Put/seqid=0 2024-11-14T19:37:29,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742408_1584 (size=12301) 2024-11-14T19:37:29,076 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/4abf523e3814402790af246f0266994e 2024-11-14T19:37:29,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/1ec75b593436408c8081bc4f9aa165fe is 50, key is test_row_0/C:col10/1731613047468/Put/seqid=0 2024-11-14T19:37:29,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742409_1585 (size=12301) 2024-11-14T19:37:29,101 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/1ec75b593436408c8081bc4f9aa165fe 2024-11-14T19:37:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7b4c055f949d49578a79ae148abaf45a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7b4c055f949d49578a79ae148abaf45a 2024-11-14T19:37:29,110 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7b4c055f949d49578a79ae148abaf45a, entries=150, sequenceid=306, filesize=30.5 K 2024-11-14T19:37:29,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/4abf523e3814402790af246f0266994e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4abf523e3814402790af246f0266994e 2024-11-14T19:37:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,116 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4abf523e3814402790af246f0266994e, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T19:37:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/1ec75b593436408c8081bc4f9aa165fe as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/1ec75b593436408c8081bc4f9aa165fe 2024-11-14T19:37:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,120 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/1ec75b593436408c8081bc4f9aa165fe, entries=150, sequenceid=306, filesize=12.0 K 2024-11-14T19:37:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,121 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 12ed89d0dac4353a0596c5193935d87c in 213ms, sequenceid=306, compaction requested=false 2024-11-14T19:37:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-14T19:37:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-14T19:37:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-14T19:37:29,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5960 sec 2024-11-14T19:37:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,134 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.6090 sec 2024-11-14T19:37:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,283 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#482 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,284 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/373a0bb2144d413a9734db964670b6cb is 175, key is test_row_0/A:col10/1731613046333/Put/seqid=0 2024-11-14T19:37:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742410_1586 (size=31835) 2024-11-14T19:37:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,300 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/373a0bb2144d413a9734db964670b6cb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/373a0bb2144d413a9734db964670b6cb 2024-11-14T19:37:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,307 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into 373a0bb2144d413a9734db964670b6cb(size=31.1 K), total size for store is 61.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:29,307 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:29,307 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=13, startTime=1731613048783; duration=0sec 2024-11-14T19:37:29,307 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:29,307 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,382 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/84bce474457449f8bb49589f02c1fc9a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/84bce474457449f8bb49589f02c1fc9a 2024-11-14T19:37:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,387 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 84bce474457449f8bb49589f02c1fc9a(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:29,387 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:29,387 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=13, startTime=1731613048784; duration=0sec 2024-11-14T19:37:29,387 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,387 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-14T19:37:29,674 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:29,674 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-14T19:37:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T19:37:29,677 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,678 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:29,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-14T19:37:29,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:29,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:29,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:29,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:29,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:29,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141575805de62948f3bdb9fd7863ed62c6_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613049674/Put/seqid=0 2024-11-14T19:37:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:29,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742411_1587 (size=17534) 2024-11-14T19:37:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T19:37:29,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613049781, max timestamp=1731613049802Num uniques 859; 2024-11-14T19:37:29,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:29,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613049781, max timestamp=1731613049802Num uniques 859; 2024-11-14T19:37:29,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613049781, max timestamp=1731613049802Num uniques 859; 2024-11-14T19:37:29,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:29,805 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:29,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:29,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:29,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:29,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:29,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613109881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:29,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:29,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613109882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:29,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:29,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:29,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:29,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:29,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:29,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T19:37:30,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613109994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613110004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:30,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,181 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:30,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411141575805de62948f3bdb9fd7863ed62c6_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141575805de62948f3bdb9fd7863ed62c6_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:30,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7a10a7856a954a8ea2850091ba8dbb2e, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:30,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7a10a7856a954a8ea2850091ba8dbb2e is 175, key is test_row_0/A:col10/1731613049674/Put/seqid=0 2024-11-14T19:37:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613110213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,215 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742412_1588 (size=48639) 2024-11-14T19:37:30,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7a10a7856a954a8ea2850091ba8dbb2e 2024-11-14T19:37:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613110223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/9cd99dbb91a34c50956d32f8524923fc is 50, key is test_row_0/B:col10/1731613049674/Put/seqid=0 2024-11-14T19:37:30,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742413_1589 (size=12301) 2024-11-14T19:37:30,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T19:37:30,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:30,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:30,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613110514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,520 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613110523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613110533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,537 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613110543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613110543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,548 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:30,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:30,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:30,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:30,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/9cd99dbb91a34c50956d32f8524923fc 2024-11-14T19:37:30,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/7f42d84e8fae4801a1dd36409e84cb69 is 50, key is test_row_0/C:col10/1731613049674/Put/seqid=0 2024-11-14T19:37:30,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742414_1590 (size=12301) 2024-11-14T19:37:30,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/7f42d84e8fae4801a1dd36409e84cb69 2024-11-14T19:37:30,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/7a10a7856a954a8ea2850091ba8dbb2e as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7a10a7856a954a8ea2850091ba8dbb2e 2024-11-14T19:37:30,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7a10a7856a954a8ea2850091ba8dbb2e, entries=250, sequenceid=318, filesize=47.5 K 2024-11-14T19:37:30,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/9cd99dbb91a34c50956d32f8524923fc as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9cd99dbb91a34c50956d32f8524923fc 2024-11-14T19:37:30,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9cd99dbb91a34c50956d32f8524923fc, entries=150, sequenceid=318, filesize=12.0 K 2024-11-14T19:37:30,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/7f42d84e8fae4801a1dd36409e84cb69 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/7f42d84e8fae4801a1dd36409e84cb69 2024-11-14T19:37:30,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:30,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:30,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T19:37:30,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/7f42d84e8fae4801a1dd36409e84cb69, entries=150, sequenceid=318, filesize=12.0 K 2024-11-14T19:37:30,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 12ed89d0dac4353a0596c5193935d87c in 1069ms, sequenceid=318, compaction requested=true 2024-11-14T19:37:30,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:30,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:30,756 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:30,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:30,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:30,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:30,756 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:30,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:30,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:30,757 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:30,757 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:30,757 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,757 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/373a0bb2144d413a9734db964670b6cb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7b4c055f949d49578a79ae148abaf45a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7a10a7856a954a8ea2850091ba8dbb2e] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=109.1 K 2024-11-14T19:37:30,757 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,757 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/373a0bb2144d413a9734db964670b6cb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7b4c055f949d49578a79ae148abaf45a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7a10a7856a954a8ea2850091ba8dbb2e] 2024-11-14T19:37:30,757 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:30,758 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:30,758 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,758 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bc686062299a4c82aadbb082e4e8dedf, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4abf523e3814402790af246f0266994e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9cd99dbb91a34c50956d32f8524923fc] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.6 K 2024-11-14T19:37:30,758 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 373a0bb2144d413a9734db964670b6cb, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731613046333 2024-11-14T19:37:30,759 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting bc686062299a4c82aadbb082e4e8dedf, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731613046333 2024-11-14T19:37:30,760 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 4abf523e3814402790af246f0266994e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731613047453 2024-11-14T19:37:30,760 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cd99dbb91a34c50956d32f8524923fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731613049674 2024-11-14T19:37:30,761 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b4c055f949d49578a79ae148abaf45a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731613047453 2024-11-14T19:37:30,761 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a10a7856a954a8ea2850091ba8dbb2e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731613049662 2024-11-14T19:37:30,780 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:30,781 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:30,781 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/bfc5375a4e1940ada332c65f291da68d is 50, key is test_row_0/B:col10/1731613049674/Put/seqid=0 2024-11-14T19:37:30,791 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241114a8b52292582141bab94413e062e1d848_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:30,793 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241114a8b52292582141bab94413e062e1d848_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:30,793 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114a8b52292582141bab94413e062e1d848_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T19:37:30,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742415_1591 (size=12983) 2024-11-14T19:37:30,841 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/bfc5375a4e1940ada332c65f291da68d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bfc5375a4e1940ada332c65f291da68d 2024-11-14T19:37:30,847 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into bfc5375a4e1940ada332c65f291da68d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:30,847 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:30,847 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=13, startTime=1731613050756; duration=0sec 2024-11-14T19:37:30,847 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:30,847 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:30,847 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:30,848 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:30,849 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:30,849 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,849 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/84bce474457449f8bb49589f02c1fc9a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/1ec75b593436408c8081bc4f9aa165fe, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/7f42d84e8fae4801a1dd36409e84cb69] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.6 K 2024-11-14T19:37:30,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742416_1592 (size=4469) 2024-11-14T19:37:30,850 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 84bce474457449f8bb49589f02c1fc9a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731613046333 2024-11-14T19:37:30,850 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ec75b593436408c8081bc4f9aa165fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731613047453 2024-11-14T19:37:30,851 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f42d84e8fae4801a1dd36409e84cb69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731613049674 2024-11-14T19:37:30,856 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#492 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:30,856 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/8db41b4c9ac649b68be42e2105a1e32b is 175, key is test_row_0/A:col10/1731613049674/Put/seqid=0 2024-11-14T19:37:30,883 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:30,884 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/360571cdfc9e4605bad7bf50b308ebbb is 50, key is test_row_0/C:col10/1731613049674/Put/seqid=0 2024-11-14T19:37:30,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742417_1593 (size=31937) 2024-11-14T19:37:30,895 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/8db41b4c9ac649b68be42e2105a1e32b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8db41b4c9ac649b68be42e2105a1e32b 2024-11-14T19:37:30,902 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into 8db41b4c9ac649b68be42e2105a1e32b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:30,902 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:30,902 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=13, startTime=1731613050756; duration=0sec 2024-11-14T19:37:30,902 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:30,902 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:30,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-14T19:37:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:30,905 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-14T19:37:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:30,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:30,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742418_1594 (size=12983) 2024-11-14T19:37:30,937 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/360571cdfc9e4605bad7bf50b308ebbb as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/360571cdfc9e4605bad7bf50b308ebbb 2024-11-14T19:37:30,946 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 360571cdfc9e4605bad7bf50b308ebbb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:30,946 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:30,946 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=13, startTime=1731613050756; duration=0sec 2024-11-14T19:37:30,947 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:30,947 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:30,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e3089c88ddbb4f728fe3c176cb181ac8_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613049880/Put/seqid=0 2024-11-14T19:37:30,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742419_1595 (size=12454) 2024-11-14T19:37:31,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:31,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613111057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613111057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613111164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613111173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,178 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613111372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613111383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:31,391 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114e3089c88ddbb4f728fe3c176cb181ac8_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e3089c88ddbb4f728fe3c176cb181ac8_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:31,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/33f8dd1c5b3e42f29abb1cfa170f8ad6, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:31,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/33f8dd1c5b3e42f29abb1cfa170f8ad6 is 175, key is test_row_0/A:col10/1731613049880/Put/seqid=0 2024-11-14T19:37:31,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742420_1596 (size=31255) 2024-11-14T19:37:31,406 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/33f8dd1c5b3e42f29abb1cfa170f8ad6 2024-11-14T19:37:31,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/cd6651d9f8d54e10bdaab8f15e249900 is 50, key is test_row_0/B:col10/1731613049880/Put/seqid=0 2024-11-14T19:37:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742421_1597 (size=12301) 2024-11-14T19:37:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613111683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613111693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T19:37:31,901 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/cd6651d9f8d54e10bdaab8f15e249900 2024-11-14T19:37:31,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/9d481cb73de24cf991695f636a5b4631 is 50, key is test_row_0/C:col10/1731613049880/Put/seqid=0 2024-11-14T19:37:31,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742422_1598 (size=12301) 2024-11-14T19:37:32,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613112193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:32,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613112203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:32,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:32,360 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/9d481cb73de24cf991695f636a5b4631 2024-11-14T19:37:32,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/33f8dd1c5b3e42f29abb1cfa170f8ad6 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33f8dd1c5b3e42f29abb1cfa170f8ad6 2024-11-14T19:37:32,373 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33f8dd1c5b3e42f29abb1cfa170f8ad6, entries=150, sequenceid=357, filesize=30.5 K 2024-11-14T19:37:32,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/cd6651d9f8d54e10bdaab8f15e249900 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/cd6651d9f8d54e10bdaab8f15e249900 2024-11-14T19:37:32,379 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/cd6651d9f8d54e10bdaab8f15e249900, entries=150, sequenceid=357, filesize=12.0 K 2024-11-14T19:37:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/9d481cb73de24cf991695f636a5b4631 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/9d481cb73de24cf991695f636a5b4631 2024-11-14T19:37:32,384 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/9d481cb73de24cf991695f636a5b4631, entries=150, sequenceid=357, filesize=12.0 K 2024-11-14T19:37:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,387 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 12ed89d0dac4353a0596c5193935d87c in 1482ms, sequenceid=357, compaction requested=false 2024-11-14T19:37:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:32,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-14T19:37:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-14T19:37:32,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-14T19:37:32,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7110 sec 2024-11-14T19:37:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7170 sec 2024-11-14T19:37:32,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:32,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:33,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-14T19:37:33,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:33,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:33,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:33,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:33,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:33,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ee63fe30c4624c9c9e3611d7b07cfe73_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613053205/Put/seqid=0 2024-11-14T19:37:33,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742423_1599 (size=12454) 2024-11-14T19:37:33,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:33,273 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ee63fe30c4624c9c9e3611d7b07cfe73_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee63fe30c4624c9c9e3611d7b07cfe73_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:33,273 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bd150c4ea5894d56bb0f5d459dadfd09, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:33,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bd150c4ea5894d56bb0f5d459dadfd09 is 175, key is test_row_0/A:col10/1731613053205/Put/seqid=0 2024-11-14T19:37:33,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742424_1600 (size=31251) 2024-11-14T19:37:33,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731613053289, max timestamp=1731613053307Num uniques 859; 2024-11-14T19:37:33,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-14T19:37:33,311 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bd150c4ea5894d56bb0f5d459dadfd09 2024-11-14T19:37:33,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613053296, max timestamp=1731613053307Num uniques 859; 2024-11-14T19:37:33,313 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:33,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731613053296, max timestamp=1731613053307Num uniques 859; 2024-11-14T19:37:33,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/867b237d0fa7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-14T19:37:33,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/f2a8e5cf595d4c638dfc6b6817ba4f0d is 50, key is test_row_0/B:col10/1731613053205/Put/seqid=0 2024-11-14T19:37:33,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742425_1601 (size=9857) 2024-11-14T19:37:33,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/f2a8e5cf595d4c638dfc6b6817ba4f0d 2024-11-14T19:37:33,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/c77880dc5146405a976ba392aa0977bc is 50, key is test_row_0/C:col10/1731613053205/Put/seqid=0 2024-11-14T19:37:33,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742426_1602 (size=9857) 2024-11-14T19:37:33,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613113424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:33,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613113426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:33,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:33,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613113543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:33,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613113544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:33,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613113753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:33,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613113754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:33,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/c77880dc5146405a976ba392aa0977bc 2024-11-14T19:37:33,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/bd150c4ea5894d56bb0f5d459dadfd09 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bd150c4ea5894d56bb0f5d459dadfd09 2024-11-14T19:37:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-14T19:37:33,834 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:33,835 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-14T19:37:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-14T19:37:33,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bd150c4ea5894d56bb0f5d459dadfd09, entries=150, sequenceid=369, filesize=30.5 K 2024-11-14T19:37:33,837 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-14T19:37:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T19:37:33,838 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T19:37:33,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/f2a8e5cf595d4c638dfc6b6817ba4f0d as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2a8e5cf595d4c638dfc6b6817ba4f0d 2024-11-14T19:37:33,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T19:37:33,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2a8e5cf595d4c638dfc6b6817ba4f0d, entries=100, sequenceid=369, filesize=9.6 K 2024-11-14T19:37:33,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/c77880dc5146405a976ba392aa0977bc as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/c77880dc5146405a976ba392aa0977bc 2024-11-14T19:37:33,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/c77880dc5146405a976ba392aa0977bc, entries=100, sequenceid=369, filesize=9.6 K 2024-11-14T19:37:33,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 12ed89d0dac4353a0596c5193935d87c in 646ms, sequenceid=369, compaction requested=true 2024-11-14T19:37:33,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:33,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:33,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:33,853 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:33,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:33,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:33,853 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:33,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:33,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:33,854 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:33,854 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:33,854 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:33,854 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8db41b4c9ac649b68be42e2105a1e32b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33f8dd1c5b3e42f29abb1cfa170f8ad6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bd150c4ea5894d56bb0f5d459dadfd09] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=92.2 K 2024-11-14T19:37:33,854 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:33,862 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8db41b4c9ac649b68be42e2105a1e32b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33f8dd1c5b3e42f29abb1cfa170f8ad6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bd150c4ea5894d56bb0f5d459dadfd09] 2024-11-14T19:37:33,863 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:33,863 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:33,863 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:33,863 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bfc5375a4e1940ada332c65f291da68d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/cd6651d9f8d54e10bdaab8f15e249900, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2a8e5cf595d4c638dfc6b6817ba4f0d] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=34.3 K 2024-11-14T19:37:33,864 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8db41b4c9ac649b68be42e2105a1e32b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731613049674 2024-11-14T19:37:33,864 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc5375a4e1940ada332c65f291da68d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731613049674 2024-11-14T19:37:33,864 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33f8dd1c5b3e42f29abb1cfa170f8ad6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731613049856 2024-11-14T19:37:33,864 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting cd6651d9f8d54e10bdaab8f15e249900, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731613049856 2024-11-14T19:37:33,864 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd150c4ea5894d56bb0f5d459dadfd09, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731613051054 2024-11-14T19:37:33,864 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a8e5cf595d4c638dfc6b6817ba4f0d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731613051054 2024-11-14T19:37:33,883 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:33,896 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:33,897 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/db8bba1f63e74afd986f6a5a6c3efe4a is 50, key is test_row_0/B:col10/1731613053205/Put/seqid=0 2024-11-14T19:37:33,901 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411149f7565dfed534305acf2ab83651ca63f_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:33,902 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411149f7565dfed534305acf2ab83651ca63f_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:33,903 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149f7565dfed534305acf2ab83651ca63f_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:33,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742427_1603 (size=13085) 2024-11-14T19:37:33,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742428_1604 (size=4469) 2024-11-14T19:37:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T19:37:33,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38601 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-14T19:37:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:33,991 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-14T19:37:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-14T19:37:34,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149f5d7daacb43489da3ef28e8603dfae6_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613053365/Put/seqid=0 2024-11-14T19:37:34,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742429_1605 (size=12454) 2024-11-14T19:37:34,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:34,038 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411149f5d7daacb43489da3ef28e8603dfae6_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149f5d7daacb43489da3ef28e8603dfae6_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:34,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/c1d88df14b544b0ab7182f16fd83ca7f, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:34,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/c1d88df14b544b0ab7182f16fd83ca7f is 175, key is test_row_0/A:col10/1731613053365/Put/seqid=0 2024-11-14T19:37:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:34,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. as already flushing 2024-11-14T19:37:34,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742430_1606 (size=31255) 2024-11-14T19:37:34,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613114121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613114121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T19:37:34,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613114234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613114234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,341 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/db8bba1f63e74afd986f6a5a6c3efe4a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/db8bba1f63e74afd986f6a5a6c3efe4a 2024-11-14T19:37:34,341 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#500 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:34,342 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/c99fbd47ecd341b0a461b270ae2e5201 is 175, key is test_row_0/A:col10/1731613053205/Put/seqid=0 2024-11-14T19:37:34,348 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into db8bba1f63e74afd986f6a5a6c3efe4a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:34,348 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:34,349 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=13, startTime=1731613053853; duration=0sec 2024-11-14T19:37:34,349 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:34,349 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:34,349 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:34,349 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:34,349 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:34,349 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:34,350 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/360571cdfc9e4605bad7bf50b308ebbb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/9d481cb73de24cf991695f636a5b4631, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/c77880dc5146405a976ba392aa0977bc] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=34.3 K 2024-11-14T19:37:34,350 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 360571cdfc9e4605bad7bf50b308ebbb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731613049674 2024-11-14T19:37:34,350 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d481cb73de24cf991695f636a5b4631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731613049856 2024-11-14T19:37:34,350 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c77880dc5146405a976ba392aa0977bc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731613051054 2024-11-14T19:37:34,369 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:34,369 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0bbb32137f104711b58e89d86df16e12 is 50, key is test_row_0/C:col10/1731613053205/Put/seqid=0 2024-11-14T19:37:34,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742431_1607 (size=32146) 2024-11-14T19:37:34,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742432_1608 (size=13085) 2024-11-14T19:37:34,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52830 deadline: 1731613114444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731613114444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,450 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T19:37:34,473 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/c1d88df14b544b0ab7182f16fd83ca7f 2024-11-14T19:37:34,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/c77768fb03054e7f95f336f52a5ed7e9 is 50, key is test_row_0/B:col10/1731613053365/Put/seqid=0 2024-11-14T19:37:34,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742433_1609 (size=12301) 2024-11-14T19:37:34,535 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/c77768fb03054e7f95f336f52a5ed7e9 2024-11-14T19:37:34,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/b6dd91acd06d4d2cb87f0100a307c1c8 is 50, key is test_row_0/C:col10/1731613053365/Put/seqid=0 2024-11-14T19:37:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52764 deadline: 1731613114563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52802 deadline: 1731613114563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T19:37:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38601 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52744 deadline: 1731613114563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:34,569 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_0' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,570 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:34,570 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 867b237d0fa7,38601,1731612889637 for 'test_row_1' in 12ed89d0dac4353a0596c5193935d87c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-14T19:37:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 , the old value is region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T19:37:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., hostname=867b237d0fa7,38601,1731612889637, seqNum=5 because the exception is null or not the one we care about 2024-11-14T19:37:34,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742434_1610 (size=12301) 2024-11-14T19:37:34,580 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/b6dd91acd06d4d2cb87f0100a307c1c8 2024-11-14T19:37:34,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/c1d88df14b544b0ab7182f16fd83ca7f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c1d88df14b544b0ab7182f16fd83ca7f 2024-11-14T19:37:34,593 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c1d88df14b544b0ab7182f16fd83ca7f, entries=150, sequenceid=405, filesize=30.5 K 2024-11-14T19:37:34,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/c77768fb03054e7f95f336f52a5ed7e9 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c77768fb03054e7f95f336f52a5ed7e9 2024-11-14T19:37:34,599 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c77768fb03054e7f95f336f52a5ed7e9, entries=150, sequenceid=405, filesize=12.0 K 2024-11-14T19:37:34,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/b6dd91acd06d4d2cb87f0100a307c1c8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/b6dd91acd06d4d2cb87f0100a307c1c8 2024-11-14T19:37:34,608 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/b6dd91acd06d4d2cb87f0100a307c1c8, entries=150, sequenceid=405, filesize=12.0 K 2024-11-14T19:37:34,611 INFO [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=60.38 KB/61830 for 12ed89d0dac4353a0596c5193935d87c in 620ms, sequenceid=405, compaction requested=false 2024-11-14T19:37:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:34,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/867b237d0fa7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-14T19:37:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-14T19:37:34,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-14T19:37:34,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 779 msec 2024-11-14T19:37:34,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 784 msec 2024-11-14T19:37:34,744 INFO [Thread-2300 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2300. 2024-11-14T19:37:34,744 DEBUG [Thread-2300 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:34,744 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,744 DEBUG [Thread-2300 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:34,746 INFO [Thread-2292 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2292. 2024-11-14T19:37:34,746 DEBUG [Thread-2292 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:34,746 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,746 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:34,747 INFO [Thread-2296 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2296. 2024-11-14T19:37:34,747 DEBUG [Thread-2296 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:34,747 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,747 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:34,748 INFO [Thread-2298 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2298. 2024-11-14T19:37:34,748 DEBUG [Thread-2298 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:34,748 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,748 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,748 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:34,749 INFO [Thread-2294 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2294. 2024-11-14T19:37:34,749 DEBUG [Thread-2294 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:34,749 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,749 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38601 {}] regionserver.HRegion(8855): Flush requested on 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:34,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-14T19:37:34,755 INFO [Thread-2281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2281. 2024-11-14T19:37:34,755 DEBUG [Thread-2281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:34,755 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:34,755 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:34,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:34,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ba287b44b3024cbca23c411d22b144ca_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613054119/Put/seqid=0 2024-11-14T19:37:34,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742435_1611 (size=12454) 2024-11-14T19:37:34,765 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-14T19:37:34,765 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:34,765 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,765 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:34,765 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:34,783 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/c99fbd47ecd341b0a461b270ae2e5201 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c99fbd47ecd341b0a461b270ae2e5201 2024-11-14T19:37:34,787 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into c99fbd47ecd341b0a461b270ae2e5201(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:34,787 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:34,787 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=13, startTime=1731613053853; duration=0sec 2024-11-14T19:37:34,787 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:34,787 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:34,802 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/0bbb32137f104711b58e89d86df16e12 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0bbb32137f104711b58e89d86df16e12 2024-11-14T19:37:34,806 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 0bbb32137f104711b58e89d86df16e12(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:34,806 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:34,806 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=13, startTime=1731613053853; duration=0sec 2024-11-14T19:37:34,806 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:34,806 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-14T19:37:34,974 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:35,164 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:35,167 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241114ba287b44b3024cbca23c411d22b144ca_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ba287b44b3024cbca23c411d22b144ca_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:35,167 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0cd9b30518ef466d92e3d1533a74485c, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:35,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0cd9b30518ef466d92e3d1533a74485c is 175, key is test_row_0/A:col10/1731613054119/Put/seqid=0 2024-11-14T19:37:35,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742436_1612 (size=31255) 2024-11-14T19:37:35,572 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0cd9b30518ef466d92e3d1533a74485c 2024-11-14T19:37:35,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/124f96d1a9764022a3ac67b3e11628ac is 50, key is test_row_0/B:col10/1731613054119/Put/seqid=0 2024-11-14T19:37:35,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742437_1613 (size=12301) 2024-11-14T19:37:35,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/124f96d1a9764022a3ac67b3e11628ac 2024-11-14T19:37:36,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a4f00f1638ec4dd6a0796bb000b6d288 is 50, key is test_row_0/C:col10/1731613054119/Put/seqid=0 2024-11-14T19:37:36,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742438_1614 (size=12301) 2024-11-14T19:37:36,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a4f00f1638ec4dd6a0796bb000b6d288 2024-11-14T19:37:36,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/0cd9b30518ef466d92e3d1533a74485c as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0cd9b30518ef466d92e3d1533a74485c 2024-11-14T19:37:36,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0cd9b30518ef466d92e3d1533a74485c, entries=150, sequenceid=419, filesize=30.5 K 2024-11-14T19:37:36,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/124f96d1a9764022a3ac67b3e11628ac as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/124f96d1a9764022a3ac67b3e11628ac 2024-11-14T19:37:36,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/124f96d1a9764022a3ac67b3e11628ac, entries=150, sequenceid=419, filesize=12.0 K 2024-11-14T19:37:36,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/a4f00f1638ec4dd6a0796bb000b6d288 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a4f00f1638ec4dd6a0796bb000b6d288 2024-11-14T19:37:36,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a4f00f1638ec4dd6a0796bb000b6d288, entries=150, sequenceid=419, filesize=12.0 K 2024-11-14T19:37:36,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for 12ed89d0dac4353a0596c5193935d87c in 1676ms, sequenceid=419, compaction requested=true 2024-11-14T19:37:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:A, priority=-2147483648, current under compaction store size is 1 2024-11-14T19:37:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:36,430 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:B, priority=-2147483648, current under compaction store size is 2 2024-11-14T19:37:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:36,430 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 12ed89d0dac4353a0596c5193935d87c:C, priority=-2147483648, current under compaction store size is 3 2024-11-14T19:37:36,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:36,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94656 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:36,431 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:36,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/A is initiating minor compaction (all files) 2024-11-14T19:37:36,431 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/B is initiating minor compaction (all files) 2024-11-14T19:37:36,431 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/A in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:36,431 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/B in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:36,431 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/db8bba1f63e74afd986f6a5a6c3efe4a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c77768fb03054e7f95f336f52a5ed7e9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/124f96d1a9764022a3ac67b3e11628ac] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.8 K 2024-11-14T19:37:36,431 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c99fbd47ecd341b0a461b270ae2e5201, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c1d88df14b544b0ab7182f16fd83ca7f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0cd9b30518ef466d92e3d1533a74485c] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=92.4 K 2024-11-14T19:37:36,431 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:36,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. files: [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c99fbd47ecd341b0a461b270ae2e5201, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c1d88df14b544b0ab7182f16fd83ca7f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0cd9b30518ef466d92e3d1533a74485c] 2024-11-14T19:37:36,431 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting db8bba1f63e74afd986f6a5a6c3efe4a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731613049856 2024-11-14T19:37:36,431 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c99fbd47ecd341b0a461b270ae2e5201, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731613049856 2024-11-14T19:37:36,432 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting c77768fb03054e7f95f336f52a5ed7e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731613053365 2024-11-14T19:37:36,432 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1d88df14b544b0ab7182f16fd83ca7f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731613053365 2024-11-14T19:37:36,432 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 124f96d1a9764022a3ac67b3e11628ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731613054118 2024-11-14T19:37:36,432 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cd9b30518ef466d92e3d1533a74485c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731613054118 2024-11-14T19:37:36,438 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:36,439 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#B#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:36,439 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ce4ae0cb13e84c908a01d80c323d4088 is 50, key is test_row_0/B:col10/1731613054119/Put/seqid=0 2024-11-14T19:37:36,439 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111449f38d7086f24246bec7e269c0bb01c8_12ed89d0dac4353a0596c5193935d87c store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:36,441 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111449f38d7086f24246bec7e269c0bb01c8_12ed89d0dac4353a0596c5193935d87c, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:36,442 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111449f38d7086f24246bec7e269c0bb01c8_12ed89d0dac4353a0596c5193935d87c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:36,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742439_1615 (size=13187) 2024-11-14T19:37:36,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742440_1616 (size=4469) 2024-11-14T19:37:36,847 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#A#compaction#510 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:36,849 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/33bdf53303c74b2eb9c34239134c16f1 is 175, key is test_row_0/A:col10/1731613054119/Put/seqid=0 2024-11-14T19:37:36,853 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ce4ae0cb13e84c908a01d80c323d4088 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ce4ae0cb13e84c908a01d80c323d4088 2024-11-14T19:37:36,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742441_1617 (size=32141) 2024-11-14T19:37:36,858 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/B of 12ed89d0dac4353a0596c5193935d87c into ce4ae0cb13e84c908a01d80c323d4088(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:36,858 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:36,858 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/B, priority=13, startTime=1731613056430; duration=0sec 2024-11-14T19:37:36,858 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-14T19:37:36,858 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:B 2024-11-14T19:37:36,858 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T19:37:36,860 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T19:37:36,860 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1541): 12ed89d0dac4353a0596c5193935d87c/C is initiating minor compaction (all files) 2024-11-14T19:37:36,860 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 12ed89d0dac4353a0596c5193935d87c/C in TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:36,860 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0bbb32137f104711b58e89d86df16e12, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/b6dd91acd06d4d2cb87f0100a307c1c8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a4f00f1638ec4dd6a0796bb000b6d288] into tmpdir=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp, totalSize=36.8 K 2024-11-14T19:37:36,860 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bbb32137f104711b58e89d86df16e12, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731613049856 2024-11-14T19:37:36,861 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting b6dd91acd06d4d2cb87f0100a307c1c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731613053365 2024-11-14T19:37:36,861 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] compactions.Compactor(225): Compacting a4f00f1638ec4dd6a0796bb000b6d288, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731613054118 2024-11-14T19:37:36,870 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 12ed89d0dac4353a0596c5193935d87c#C#compaction#511 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T19:37:36,871 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/423711aac5c441dc855da2ec54fd6f7b is 50, key is test_row_0/C:col10/1731613054119/Put/seqid=0 2024-11-14T19:37:36,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742442_1618 (size=13187) 2024-11-14T19:37:37,267 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/33bdf53303c74b2eb9c34239134c16f1 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33bdf53303c74b2eb9c34239134c16f1 2024-11-14T19:37:37,271 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/A of 12ed89d0dac4353a0596c5193935d87c into 33bdf53303c74b2eb9c34239134c16f1(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:37,271 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:37,271 INFO [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/A, priority=13, startTime=1731613056430; duration=0sec 2024-11-14T19:37:37,271 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:37,271 DEBUG [RS:0;867b237d0fa7:38601-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:A 2024-11-14T19:37:37,278 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/423711aac5c441dc855da2ec54fd6f7b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/423711aac5c441dc855da2ec54fd6f7b 2024-11-14T19:37:37,283 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 12ed89d0dac4353a0596c5193935d87c/C of 12ed89d0dac4353a0596c5193935d87c into 423711aac5c441dc855da2ec54fd6f7b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T19:37:37,284 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 12ed89d0dac4353a0596c5193935d87c: 2024-11-14T19:37:37,284 INFO [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., storeName=12ed89d0dac4353a0596c5193935d87c/C, priority=13, startTime=1731613056430; duration=0sec 2024-11-14T19:37:37,284 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T19:37:37,284 DEBUG [RS:0;867b237d0fa7:38601-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 12ed89d0dac4353a0596c5193935d87c:C 2024-11-14T19:37:44,585 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-14T19:37:44,585 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:44,585 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,585 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:44,604 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-14T19:37:44,604 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:44,604 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,604 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,604 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:44,624 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-11-14T19:37:44,624 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-14T19:37:44,624 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,624 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 146 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2280 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2207 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2121 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2286 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2211 2024-11-14T19:37:44,625 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-14T19:37:44,625 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:37:44,625 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:37:44,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:44,625 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:44,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-14T19:37:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T19:37:44,628 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613064628"}]},"ts":"1731613064628"} 2024-11-14T19:37:44,629 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-14T19:37:44,629 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-14T19:37:44,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-14T19:37:44,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, UNASSIGN}] 2024-11-14T19:37:44,632 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, UNASSIGN 2024-11-14T19:37:44,632 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=CLOSING, regionLocation=867b237d0fa7,38601,1731612889637 2024-11-14T19:37:44,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, UNASSIGN because future has completed 2024-11-14T19:37:44,633 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T19:37:44,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637}] 2024-11-14T19:37:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T19:37:44,785 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:44,785 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T19:37:44,785 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 12ed89d0dac4353a0596c5193935d87c, disabling compactions & flushes 2024-11-14T19:37:44,785 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:44,785 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:44,785 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. after waiting 0 ms 2024-11-14T19:37:44,785 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:44,786 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 12ed89d0dac4353a0596c5193935d87c 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-14T19:37:44,786 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=A 2024-11-14T19:37:44,786 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:44,786 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=B 2024-11-14T19:37:44,786 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:44,786 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 12ed89d0dac4353a0596c5193935d87c, store=C 2024-11-14T19:37:44,786 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-14T19:37:44,790 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147a97054813bc440cbdaf4a4655914755_12ed89d0dac4353a0596c5193935d87c is 50, key is test_row_0/A:col10/1731613064584/Put/seqid=0 2024-11-14T19:37:44,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742443_1619 (size=12454) 2024-11-14T19:37:44,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T19:37:45,194 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T19:37:45,200 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411147a97054813bc440cbdaf4a4655914755_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147a97054813bc440cbdaf4a4655914755_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:45,201 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/68c6b3891c744d58864161d252bf5859, store: [table=TestAcidGuarantees family=A region=12ed89d0dac4353a0596c5193935d87c] 2024-11-14T19:37:45,202 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/68c6b3891c744d58864161d252bf5859 is 175, key is test_row_0/A:col10/1731613064584/Put/seqid=0 2024-11-14T19:37:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742444_1620 (size=31255) 2024-11-14T19:37:45,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T19:37:45,641 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/68c6b3891c744d58864161d252bf5859 2024-11-14T19:37:45,646 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ba20d1fee7974129bb7fbd64d8fd92a3 is 50, key is test_row_0/B:col10/1731613064584/Put/seqid=0 2024-11-14T19:37:45,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742445_1621 (size=12301) 2024-11-14T19:37:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T19:37:46,056 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ba20d1fee7974129bb7fbd64d8fd92a3 2024-11-14T19:37:46,062 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/85c42a55f4bb4f8483f7841e044301a8 is 50, key is test_row_0/C:col10/1731613064584/Put/seqid=0 2024-11-14T19:37:46,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742446_1622 (size=12301) 2024-11-14T19:37:46,466 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/85c42a55f4bb4f8483f7841e044301a8 2024-11-14T19:37:46,469 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/A/68c6b3891c744d58864161d252bf5859 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/68c6b3891c744d58864161d252bf5859 2024-11-14T19:37:46,473 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/68c6b3891c744d58864161d252bf5859, entries=150, sequenceid=431, filesize=30.5 K 2024-11-14T19:37:46,473 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/B/ba20d1fee7974129bb7fbd64d8fd92a3 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ba20d1fee7974129bb7fbd64d8fd92a3 2024-11-14T19:37:46,476 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ba20d1fee7974129bb7fbd64d8fd92a3, entries=150, sequenceid=431, filesize=12.0 K 2024-11-14T19:37:46,477 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/.tmp/C/85c42a55f4bb4f8483f7841e044301a8 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/85c42a55f4bb4f8483f7841e044301a8 2024-11-14T19:37:46,480 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/85c42a55f4bb4f8483f7841e044301a8, entries=150, sequenceid=431, filesize=12.0 K 2024-11-14T19:37:46,481 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 12ed89d0dac4353a0596c5193935d87c in 1696ms, sequenceid=431, compaction requested=false 2024-11-14T19:37:46,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5c4f35696ebb42e1877d261868d2c2db, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/efb57c24e3b24e9caec56be7ffdf757a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/dec502adcb564931b521617ee7d6d2e7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/909ae297420e46e7845447e37545ce6c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/345795b8a5bd48dfaf1cef5eb5d1af5d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/3f47325214404e8980cd6de3ade374fb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bc7cfb73c67f4793becb3fd0a8bbeb48, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/d2a08e5362624022a3ebe931ff23c8e2, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0ac0de1775764403bec8431b27e8fc53, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fa3d5d795e7e4151817cc2ff628f1ec8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5b75ecb959864d0f9c365038647dd9ba, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/21d70ce5a3a440799d722697ab01d02f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fbcaa84735344c95b972adf8e6203959, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fe5b4e1aa9044271a84d47f5765f7cbc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/e12473c3081b48eb976bbe15e2369567, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/327282b262094cf88399c42db4ccdc0a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8e5a8594203046faa7caa44e6bb2f3a1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/373a0bb2144d413a9734db964670b6cb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7b4c055f949d49578a79ae148abaf45a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7a10a7856a954a8ea2850091ba8dbb2e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8db41b4c9ac649b68be42e2105a1e32b, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33f8dd1c5b3e42f29abb1cfa170f8ad6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c99fbd47ecd341b0a461b270ae2e5201, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bd150c4ea5894d56bb0f5d459dadfd09, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c1d88df14b544b0ab7182f16fd83ca7f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0cd9b30518ef466d92e3d1533a74485c] to archive 2024-11-14T19:37:46,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:37:46,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5c4f35696ebb42e1877d261868d2c2db to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5c4f35696ebb42e1877d261868d2c2db 2024-11-14T19:37:46,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/efb57c24e3b24e9caec56be7ffdf757a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/efb57c24e3b24e9caec56be7ffdf757a 2024-11-14T19:37:46,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/dec502adcb564931b521617ee7d6d2e7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/dec502adcb564931b521617ee7d6d2e7 2024-11-14T19:37:46,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/909ae297420e46e7845447e37545ce6c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/909ae297420e46e7845447e37545ce6c 2024-11-14T19:37:46,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/345795b8a5bd48dfaf1cef5eb5d1af5d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/345795b8a5bd48dfaf1cef5eb5d1af5d 2024-11-14T19:37:46,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/3f47325214404e8980cd6de3ade374fb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/3f47325214404e8980cd6de3ade374fb 2024-11-14T19:37:46,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bc7cfb73c67f4793becb3fd0a8bbeb48 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bc7cfb73c67f4793becb3fd0a8bbeb48 2024-11-14T19:37:46,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/d2a08e5362624022a3ebe931ff23c8e2 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/d2a08e5362624022a3ebe931ff23c8e2 2024-11-14T19:37:46,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0ac0de1775764403bec8431b27e8fc53 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0ac0de1775764403bec8431b27e8fc53 2024-11-14T19:37:46,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fa3d5d795e7e4151817cc2ff628f1ec8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fa3d5d795e7e4151817cc2ff628f1ec8 2024-11-14T19:37:46,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5b75ecb959864d0f9c365038647dd9ba to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/5b75ecb959864d0f9c365038647dd9ba 2024-11-14T19:37:46,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/21d70ce5a3a440799d722697ab01d02f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/21d70ce5a3a440799d722697ab01d02f 2024-11-14T19:37:46,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fbcaa84735344c95b972adf8e6203959 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fbcaa84735344c95b972adf8e6203959 2024-11-14T19:37:46,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fe5b4e1aa9044271a84d47f5765f7cbc to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/fe5b4e1aa9044271a84d47f5765f7cbc 2024-11-14T19:37:46,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/e12473c3081b48eb976bbe15e2369567 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/e12473c3081b48eb976bbe15e2369567 2024-11-14T19:37:46,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/327282b262094cf88399c42db4ccdc0a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/327282b262094cf88399c42db4ccdc0a 2024-11-14T19:37:46,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8e5a8594203046faa7caa44e6bb2f3a1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8e5a8594203046faa7caa44e6bb2f3a1 2024-11-14T19:37:46,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/373a0bb2144d413a9734db964670b6cb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/373a0bb2144d413a9734db964670b6cb 2024-11-14T19:37:46,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7b4c055f949d49578a79ae148abaf45a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7b4c055f949d49578a79ae148abaf45a 2024-11-14T19:37:46,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7a10a7856a954a8ea2850091ba8dbb2e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/7a10a7856a954a8ea2850091ba8dbb2e 2024-11-14T19:37:46,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8db41b4c9ac649b68be42e2105a1e32b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/8db41b4c9ac649b68be42e2105a1e32b 2024-11-14T19:37:46,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33f8dd1c5b3e42f29abb1cfa170f8ad6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33f8dd1c5b3e42f29abb1cfa170f8ad6 2024-11-14T19:37:46,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c99fbd47ecd341b0a461b270ae2e5201 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c99fbd47ecd341b0a461b270ae2e5201 2024-11-14T19:37:46,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bd150c4ea5894d56bb0f5d459dadfd09 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/bd150c4ea5894d56bb0f5d459dadfd09 2024-11-14T19:37:46,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c1d88df14b544b0ab7182f16fd83ca7f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/c1d88df14b544b0ab7182f16fd83ca7f 2024-11-14T19:37:46,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0cd9b30518ef466d92e3d1533a74485c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/0cd9b30518ef466d92e3d1533a74485c 2024-11-14T19:37:46,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/76250cbadf14464e83f4663dd2871b5c, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/0fbf2d87f0664868b41451d1f5c4f886, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/6888275980c3474faab48ad54631cba7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9fe3fa6c20f047d9a61aa7babc9e80a8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/27009b92f24540559d74205f1650442f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2b237cc46114c0384945dde7913b6b7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2a7c44d70f7b49c1acea041d30de40e1, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a119cc711f6443b8b02d90231f724c55, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/eaa4aea17fc0449f859518a45aa1b5ac, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/5cd840a0677440959c2817e74170bdbd, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4efb6d3db1fd43218cf63c5df7c98cf4, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/38921ba854b242beaa09f937dc119c5d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a360214223044f5fbf41390b64f3af3f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c9a1babbcd0f4c3483bb9b69c56ac999, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2453875028304b80bd93a93be064c76f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/1545d867a77a4335b0d71c86e12aabbf, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bc686062299a4c82aadbb082e4e8dedf, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ddf8a3a72eb34395acb7387ad79fc164, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4abf523e3814402790af246f0266994e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bfc5375a4e1940ada332c65f291da68d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9cd99dbb91a34c50956d32f8524923fc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/cd6651d9f8d54e10bdaab8f15e249900, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/db8bba1f63e74afd986f6a5a6c3efe4a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2a8e5cf595d4c638dfc6b6817ba4f0d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c77768fb03054e7f95f336f52a5ed7e9, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/124f96d1a9764022a3ac67b3e11628ac] to archive 2024-11-14T19:37:46,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:37:46,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/76250cbadf14464e83f4663dd2871b5c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/76250cbadf14464e83f4663dd2871b5c 2024-11-14T19:37:46,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/0fbf2d87f0664868b41451d1f5c4f886 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/0fbf2d87f0664868b41451d1f5c4f886 2024-11-14T19:37:46,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/6888275980c3474faab48ad54631cba7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/6888275980c3474faab48ad54631cba7 2024-11-14T19:37:46,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9fe3fa6c20f047d9a61aa7babc9e80a8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9fe3fa6c20f047d9a61aa7babc9e80a8 2024-11-14T19:37:46,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/27009b92f24540559d74205f1650442f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/27009b92f24540559d74205f1650442f 2024-11-14T19:37:46,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2b237cc46114c0384945dde7913b6b7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2b237cc46114c0384945dde7913b6b7 2024-11-14T19:37:46,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2a7c44d70f7b49c1acea041d30de40e1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2a7c44d70f7b49c1acea041d30de40e1 2024-11-14T19:37:46,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a119cc711f6443b8b02d90231f724c55 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a119cc711f6443b8b02d90231f724c55 2024-11-14T19:37:46,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/eaa4aea17fc0449f859518a45aa1b5ac to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/eaa4aea17fc0449f859518a45aa1b5ac 2024-11-14T19:37:46,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/5cd840a0677440959c2817e74170bdbd to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/5cd840a0677440959c2817e74170bdbd 2024-11-14T19:37:46,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4efb6d3db1fd43218cf63c5df7c98cf4 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4efb6d3db1fd43218cf63c5df7c98cf4 2024-11-14T19:37:46,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/38921ba854b242beaa09f937dc119c5d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/38921ba854b242beaa09f937dc119c5d 2024-11-14T19:37:46,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a360214223044f5fbf41390b64f3af3f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/a360214223044f5fbf41390b64f3af3f 2024-11-14T19:37:46,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c9a1babbcd0f4c3483bb9b69c56ac999 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c9a1babbcd0f4c3483bb9b69c56ac999 2024-11-14T19:37:46,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2453875028304b80bd93a93be064c76f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/2453875028304b80bd93a93be064c76f 2024-11-14T19:37:46,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/1545d867a77a4335b0d71c86e12aabbf to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/1545d867a77a4335b0d71c86e12aabbf 2024-11-14T19:37:46,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bc686062299a4c82aadbb082e4e8dedf to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bc686062299a4c82aadbb082e4e8dedf 2024-11-14T19:37:46,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ddf8a3a72eb34395acb7387ad79fc164 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ddf8a3a72eb34395acb7387ad79fc164 2024-11-14T19:37:46,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4abf523e3814402790af246f0266994e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/4abf523e3814402790af246f0266994e 2024-11-14T19:37:46,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bfc5375a4e1940ada332c65f291da68d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/bfc5375a4e1940ada332c65f291da68d 2024-11-14T19:37:46,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9cd99dbb91a34c50956d32f8524923fc to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/9cd99dbb91a34c50956d32f8524923fc 2024-11-14T19:37:46,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/cd6651d9f8d54e10bdaab8f15e249900 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/cd6651d9f8d54e10bdaab8f15e249900 2024-11-14T19:37:46,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/db8bba1f63e74afd986f6a5a6c3efe4a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/db8bba1f63e74afd986f6a5a6c3efe4a 2024-11-14T19:37:46,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2a8e5cf595d4c638dfc6b6817ba4f0d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/f2a8e5cf595d4c638dfc6b6817ba4f0d 2024-11-14T19:37:46,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c77768fb03054e7f95f336f52a5ed7e9 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/c77768fb03054e7f95f336f52a5ed7e9 2024-11-14T19:37:46,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/124f96d1a9764022a3ac67b3e11628ac to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/124f96d1a9764022a3ac67b3e11628ac 2024-11-14T19:37:46,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8397fa4eef1a474991529708f892fa0e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a2c99c2139754560ada9ac6a1e15f5a6, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/24d7dd906e3a472089617b6a59f4a585, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a77597d4b6d648e395b1aa6756b1003f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/d5a647a9d40b4400a47cb5939d4453aa, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/3e3354d408ee43288770a1f55de7e0ac, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/939f488e5b5c41c1b618f293dc993004, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0058ac3c2f974024888cd4fccc51695d, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/826b1aa1da4448c0ae60a3e3382b7d02, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/ee30f89f986c4f3c8b9cecf7e94bfc6f, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/2d0740c329714d57bdddaa5d61105e06, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/70c5e91d7ade472396e08745c73d541e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/95216910f7114619b45733ee6cf89bde, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/4bd05868c8304a4e869febed74595c87, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/f5682dfe38af4943a08e0ba20aad631e, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0e89416bde5b49aabf377d5d19d83012, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/84bce474457449f8bb49589f02c1fc9a, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8737dd7859c246c99080d521a69c0bc7, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/1ec75b593436408c8081bc4f9aa165fe, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/360571cdfc9e4605bad7bf50b308ebbb, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/7f42d84e8fae4801a1dd36409e84cb69, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/9d481cb73de24cf991695f636a5b4631, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0bbb32137f104711b58e89d86df16e12, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/c77880dc5146405a976ba392aa0977bc, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/b6dd91acd06d4d2cb87f0100a307c1c8, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a4f00f1638ec4dd6a0796bb000b6d288] to archive 2024-11-14T19:37:46,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T19:37:46,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8397fa4eef1a474991529708f892fa0e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8397fa4eef1a474991529708f892fa0e 2024-11-14T19:37:46,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a2c99c2139754560ada9ac6a1e15f5a6 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a2c99c2139754560ada9ac6a1e15f5a6 2024-11-14T19:37:46,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/24d7dd906e3a472089617b6a59f4a585 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/24d7dd906e3a472089617b6a59f4a585 2024-11-14T19:37:46,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a77597d4b6d648e395b1aa6756b1003f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a77597d4b6d648e395b1aa6756b1003f 2024-11-14T19:37:46,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/d5a647a9d40b4400a47cb5939d4453aa to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/d5a647a9d40b4400a47cb5939d4453aa 2024-11-14T19:37:46,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/3e3354d408ee43288770a1f55de7e0ac to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/3e3354d408ee43288770a1f55de7e0ac 2024-11-14T19:37:46,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/939f488e5b5c41c1b618f293dc993004 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/939f488e5b5c41c1b618f293dc993004 2024-11-14T19:37:46,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0058ac3c2f974024888cd4fccc51695d to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0058ac3c2f974024888cd4fccc51695d 2024-11-14T19:37:46,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/826b1aa1da4448c0ae60a3e3382b7d02 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/826b1aa1da4448c0ae60a3e3382b7d02 2024-11-14T19:37:46,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/ee30f89f986c4f3c8b9cecf7e94bfc6f to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/ee30f89f986c4f3c8b9cecf7e94bfc6f 2024-11-14T19:37:46,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/2d0740c329714d57bdddaa5d61105e06 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/2d0740c329714d57bdddaa5d61105e06 2024-11-14T19:37:46,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/70c5e91d7ade472396e08745c73d541e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/70c5e91d7ade472396e08745c73d541e 2024-11-14T19:37:46,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/95216910f7114619b45733ee6cf89bde to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/95216910f7114619b45733ee6cf89bde 2024-11-14T19:37:46,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/4bd05868c8304a4e869febed74595c87 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/4bd05868c8304a4e869febed74595c87 2024-11-14T19:37:46,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/f5682dfe38af4943a08e0ba20aad631e to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/f5682dfe38af4943a08e0ba20aad631e 2024-11-14T19:37:46,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0e89416bde5b49aabf377d5d19d83012 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0e89416bde5b49aabf377d5d19d83012 2024-11-14T19:37:46,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/84bce474457449f8bb49589f02c1fc9a to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/84bce474457449f8bb49589f02c1fc9a 2024-11-14T19:37:46,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8737dd7859c246c99080d521a69c0bc7 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/8737dd7859c246c99080d521a69c0bc7 2024-11-14T19:37:46,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/1ec75b593436408c8081bc4f9aa165fe to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/1ec75b593436408c8081bc4f9aa165fe 2024-11-14T19:37:46,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/360571cdfc9e4605bad7bf50b308ebbb to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/360571cdfc9e4605bad7bf50b308ebbb 2024-11-14T19:37:46,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/7f42d84e8fae4801a1dd36409e84cb69 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/7f42d84e8fae4801a1dd36409e84cb69 2024-11-14T19:37:46,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/9d481cb73de24cf991695f636a5b4631 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/9d481cb73de24cf991695f636a5b4631 2024-11-14T19:37:46,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0bbb32137f104711b58e89d86df16e12 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/0bbb32137f104711b58e89d86df16e12 2024-11-14T19:37:46,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/c77880dc5146405a976ba392aa0977bc to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/c77880dc5146405a976ba392aa0977bc 2024-11-14T19:37:46,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/b6dd91acd06d4d2cb87f0100a307c1c8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/b6dd91acd06d4d2cb87f0100a307c1c8 2024-11-14T19:37:46,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a4f00f1638ec4dd6a0796bb000b6d288 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/a4f00f1638ec4dd6a0796bb000b6d288 2024-11-14T19:37:46,579 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/recovered.edits/434.seqid, newMaxSeqId=434, maxSeqId=4 2024-11-14T19:37:46,583 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. 2024-11-14T19:37:46,583 DEBUG [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 12ed89d0dac4353a0596c5193935d87c: Waiting for close lock at 1731613064785Running coprocessor pre-close hooks at 1731613064785Disabling compacts and flushes for region at 1731613064785Disabling writes for close at 1731613064785Obtaining lock to block concurrent updates at 1731613064786 (+1 ms)Preparing flush snapshotting stores in 12ed89d0dac4353a0596c5193935d87c at 1731613064786Finished memstore snapshotting TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731613064786Flushing stores of TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c. at 1731613064786Flushing 12ed89d0dac4353a0596c5193935d87c/A: creating writer at 1731613064786Flushing mob file 12ed89d0dac4353a0596c5193935d87c/A: appending metadata at 1731613064790 (+4 ms)Flushing mob file 12ed89d0dac4353a0596c5193935d87c/A: closing flushed file at 1731613064790Flushing 12ed89d0dac4353a0596c5193935d87c/A: appending metadata at 1731613065201 (+411 ms)Flushing 12ed89d0dac4353a0596c5193935d87c/A: closing flushed file at 1731613065202 (+1 ms)Flushing 12ed89d0dac4353a0596c5193935d87c/B: creating writer at 1731613065644 (+442 ms)Flushing 12ed89d0dac4353a0596c5193935d87c/B: appending metadata at 1731613065646 (+2 ms)Flushing 12ed89d0dac4353a0596c5193935d87c/B: closing flushed file at 1731613065646Flushing 12ed89d0dac4353a0596c5193935d87c/C: creating writer at 1731613066060 (+414 ms)Flushing 12ed89d0dac4353a0596c5193935d87c/C: appending metadata at 1731613066062 (+2 ms)Flushing 12ed89d0dac4353a0596c5193935d87c/C: closing flushed file at 1731613066062Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5807dd1b: reopening flushed file at 1731613066469 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@235bc8ea: reopening flushed file at 1731613066473 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57b2b11c: reopening flushed file at 1731613066477 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 12ed89d0dac4353a0596c5193935d87c in 1696ms, sequenceid=431, compaction requested=false at 1731613066481 (+4 ms)Writing region close event to WAL at 1731613066574 (+93 ms)Running coprocessor post-close hooks at 1731613066583 (+9 ms)Closed at 1731613066583 2024-11-14T19:37:46,585 INFO [RS_CLOSE_REGION-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,586 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=12ed89d0dac4353a0596c5193935d87c, regionState=CLOSED 2024-11-14T19:37:46,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 because future has completed 2024-11-14T19:37:46,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-14T19:37:46,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 12ed89d0dac4353a0596c5193935d87c, server=867b237d0fa7,38601,1731612889637 in 1.9560 sec 2024-11-14T19:37:46,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-14T19:37:46,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=12ed89d0dac4353a0596c5193935d87c, UNASSIGN in 1.9600 sec 2024-11-14T19:37:46,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-14T19:37:46,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9640 sec 2024-11-14T19:37:46,597 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731613066597"}]},"ts":"1731613066597"} 2024-11-14T19:37:46,599 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-14T19:37:46,599 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-14T19:37:46,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9740 sec 2024-11-14T19:37:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-14T19:37:46,774 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:46,774 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-14T19:37:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:46,776 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:46,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T19:37:46,777 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:46,779 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,781 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C, FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/recovered.edits] 2024-11-14T19:37:46,783 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33bdf53303c74b2eb9c34239134c16f1 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/33bdf53303c74b2eb9c34239134c16f1 2024-11-14T19:37:46,784 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/68c6b3891c744d58864161d252bf5859 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/A/68c6b3891c744d58864161d252bf5859 2024-11-14T19:37:46,786 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ba20d1fee7974129bb7fbd64d8fd92a3 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ba20d1fee7974129bb7fbd64d8fd92a3 2024-11-14T19:37:46,787 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ce4ae0cb13e84c908a01d80c323d4088 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/B/ce4ae0cb13e84c908a01d80c323d4088 2024-11-14T19:37:46,789 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/423711aac5c441dc855da2ec54fd6f7b to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/423711aac5c441dc855da2ec54fd6f7b 2024-11-14T19:37:46,790 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/85c42a55f4bb4f8483f7841e044301a8 to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/C/85c42a55f4bb4f8483f7841e044301a8 2024-11-14T19:37:46,792 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/recovered.edits/434.seqid to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c/recovered.edits/434.seqid 2024-11-14T19:37:46,793 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/default/TestAcidGuarantees/12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,793 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-14T19:37:46,793 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T19:37:46,794 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-14T19:37:46,797 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140e0100dcc56a44baa7aa8647f02b3ee2_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411140e0100dcc56a44baa7aa8647f02b3ee2_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,798 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141575805de62948f3bdb9fd7863ed62c6_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411141575805de62948f3bdb9fd7863ed62c6_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,799 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114305d060e79ce4591849c3be0790107f7_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114305d060e79ce4591849c3be0790107f7_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,800 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143def81e60ad14efea244b4f8df6d2757_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411143def81e60ad14efea244b4f8df6d2757_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,800 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144823755deb7b44fd8b0646e52ed9e3ae_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411144823755deb7b44fd8b0646e52ed9e3ae_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,801 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145005c5fc648c41aa84ed805bae4f65f7_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411145005c5fc648c41aa84ed805bae4f65f7_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,802 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114632e390f22b443c7a5771ce5ee88a904_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114632e390f22b443c7a5771ce5ee88a904_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,803 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114647947190f6b4a46ba1a13f323bca286_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114647947190f6b4a46ba1a13f323bca286_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,806 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114669c95344e2d46769e0a334acd642c38_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114669c95344e2d46769e0a334acd642c38_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,808 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147a97054813bc440cbdaf4a4655914755_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411147a97054813bc440cbdaf4a4655914755_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,809 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148bb38b27db16470bab07bfef8d1bfca3_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411148bb38b27db16470bab07bfef8d1bfca3_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,810 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114982f45a84bfe44a9a5c87ba3aac21524_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114982f45a84bfe44a9a5c87ba3aac21524_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,810 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149f5d7daacb43489da3ef28e8603dfae6_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411149f5d7daacb43489da3ef28e8603dfae6_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,811 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a58a26b2c9d94a02ae6d462a545b4915_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114a58a26b2c9d94a02ae6d462a545b4915_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,812 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b25b334f8a8e422db833a7c46a671f5c_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b25b334f8a8e422db833a7c46a671f5c_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,813 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3a7894e3c4242de8922d895cdc7c6a3_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114b3a7894e3c4242de8922d895cdc7c6a3_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,813 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ba287b44b3024cbca23c411d22b144ca_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ba287b44b3024cbca23c411d22b144ca_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,814 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c8242525b3f54da5a439d83cc9c9f54a_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114c8242525b3f54da5a439d83cc9c9f54a_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,815 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e3089c88ddbb4f728fe3c176cb181ac8_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114e3089c88ddbb4f728fe3c176cb181ac8_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,816 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee63fe30c4624c9c9e3611d7b07cfe73_12ed89d0dac4353a0596c5193935d87c to hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241114ee63fe30c4624c9c9e3611d7b07cfe73_12ed89d0dac4353a0596c5193935d87c 2024-11-14T19:37:46,816 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-14T19:37:46,818 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:46,820 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-14T19:37:46,821 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-14T19:37:46,823 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:46,823 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-14T19:37:46,823 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731613066823"}]},"ts":"9223372036854775807"} 2024-11-14T19:37:46,825 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-14T19:37:46,825 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 12ed89d0dac4353a0596c5193935d87c, NAME => 'TestAcidGuarantees,,1731613031583.12ed89d0dac4353a0596c5193935d87c.', STARTKEY => '', ENDKEY => ''}] 2024-11-14T19:37:46,825 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-14T19:37:46,825 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731613066825"}]},"ts":"9223372036854775807"} 2024-11-14T19:37:46,826 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-14T19:37:46,827 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-14T19:37:46,829 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-14T19:37:46,861 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T19:37:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38019 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-14T19:37:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-14T19:37:46,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-14T19:37:46,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 251), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=579 (was 599), ProcessCount=11 (was 11), AvailableMemoryMB=6621 (was 5452) - AvailableMemoryMB LEAK? - 2024-11-14T19:37:46,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T19:37:46,895 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:37:46,895 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:37:46,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:46,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:46,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T19:37:46,895 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T19:37:46,895 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-11-14T19:37:46,896 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=867b237d0fa7,38019,1731612888274 2024-11-14T19:37:46,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T19:37:46,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T19:37:46,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:37:46,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:37:46,948 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T19:37:46,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T19:37:46,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:37:46,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:46,949 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T19:37:46,949 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T19:37:46,950 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '867b237d0fa7,38601,1731612889637' ***** 2024-11-14T19:37:46,950 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T19:37:46,950 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T19:37:46,950 INFO [RS:0;867b237d0fa7:38601 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T19:37:46,950 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T19:37:46,950 INFO [RS:0;867b237d0fa7:38601 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T19:37:46,950 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(959): stopping server 867b237d0fa7,38601,1731612889637 2024-11-14T19:37:46,950 INFO [RS:0;867b237d0fa7:38601 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T19:37:46,950 INFO [RS:0;867b237d0fa7:38601 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;867b237d0fa7:38601. 2024-11-14T19:37:46,950 DEBUG [RS:0;867b237d0fa7:38601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T19:37:46,950 DEBUG [RS:0;867b237d0fa7:38601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:46,951 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T19:37:46,951 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T19:37:46,951 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T19:37:46,951 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T19:37:46,951 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T19:37:46,951 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T19:37:46,952 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T19:37:46,952 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T19:37:46,952 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T19:37:46,952 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T19:37:46,952 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T19:37:46,952 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T19:37:46,952 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-14T19:37:46,974 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/info/fb4e4e28244a4c6183abe2aa1f665cba is 95, key is TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae./info:/1731612918117/DeleteFamily/seqid=0 2024-11-14T19:37:46,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742447_1623 (size=6514) 2024-11-14T19:37:46,989 INFO [regionserver/867b237d0fa7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T19:37:47,152 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T19:37:47,352 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T19:37:47,378 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/info/fb4e4e28244a4c6183abe2aa1f665cba 2024-11-14T19:37:47,400 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/ns/11c9b771d0e44ab5abe1c616d24b303f is 93, key is TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae./ns:/1731612918058/DeleteFamily/seqid=0 2024-11-14T19:37:47,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742448_1624 (size=5906) 2024-11-14T19:37:47,404 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/ns/11c9b771d0e44ab5abe1c616d24b303f 2024-11-14T19:37:47,421 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/rep_barrier/67427c06bd124560b1f09a9f3c39bc0a is 102, key is TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae./rep_barrier:/1731612918058/DeleteFamily/seqid=0 2024-11-14T19:37:47,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742449_1625 (size=6025) 2024-11-14T19:37:47,552 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T19:37:47,652 INFO [regionserver/867b237d0fa7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T19:37:47,652 INFO [regionserver/867b237d0fa7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T19:37:47,752 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T19:37:47,826 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/rep_barrier/67427c06bd124560b1f09a9f3c39bc0a 2024-11-14T19:37:47,849 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/table/d3ad6628e73740088e0f93974b273efa is 96, key is TestAcidGuarantees,,1731612894297.21b827b5bf7b423416191650584b91ae./table:/1731612918058/DeleteFamily/seqid=0 2024-11-14T19:37:47,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742450_1626 (size=5936) 2024-11-14T19:37:47,953 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T19:37:48,153 DEBUG [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T19:37:48,253 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/table/d3ad6628e73740088e0f93974b273efa 2024-11-14T19:37:48,256 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/info/fb4e4e28244a4c6183abe2aa1f665cba as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/info/fb4e4e28244a4c6183abe2aa1f665cba 2024-11-14T19:37:48,259 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/info/fb4e4e28244a4c6183abe2aa1f665cba, entries=12, sequenceid=90, filesize=6.4 K 2024-11-14T19:37:48,260 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/ns/11c9b771d0e44ab5abe1c616d24b303f as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/ns/11c9b771d0e44ab5abe1c616d24b303f 2024-11-14T19:37:48,263 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/ns/11c9b771d0e44ab5abe1c616d24b303f, entries=8, sequenceid=90, filesize=5.8 K 2024-11-14T19:37:48,263 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/rep_barrier/67427c06bd124560b1f09a9f3c39bc0a as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/rep_barrier/67427c06bd124560b1f09a9f3c39bc0a 2024-11-14T19:37:48,267 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/rep_barrier/67427c06bd124560b1f09a9f3c39bc0a, entries=6, sequenceid=90, filesize=5.9 K 2024-11-14T19:37:48,267 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/.tmp/table/d3ad6628e73740088e0f93974b273efa as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/table/d3ad6628e73740088e0f93974b273efa 2024-11-14T19:37:48,270 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/table/d3ad6628e73740088e0f93974b273efa, entries=7, sequenceid=90, filesize=5.8 K 2024-11-14T19:37:48,271 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1319ms, sequenceid=90, compaction requested=false 2024-11-14T19:37:48,275 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-14T19:37:48,276 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T19:37:48,276 INFO [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T19:37:48,276 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731613066951Running coprocessor pre-close hooks at 1731613066952 (+1 ms)Disabling compacts and flushes for region at 1731613066952Disabling writes for close at 1731613066952Obtaining lock to block concurrent updates at 1731613066952Preparing flush snapshotting stores in 1588230740 at 1731613066952Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731613066952Flushing stores of hbase:meta,,1.1588230740 at 1731613066953 (+1 ms)Flushing 1588230740/info: creating writer at 1731613066953Flushing 1588230740/info: appending metadata at 1731613066973 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731613066973Flushing 1588230740/ns: creating writer at 1731613067381 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731613067399 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731613067399Flushing 1588230740/rep_barrier: creating writer at 1731613067407 (+8 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731613067421 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731613067421Flushing 1588230740/table: creating writer at 1731613067829 (+408 ms)Flushing 1588230740/table: appending metadata at 1731613067848 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731613067848Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a1f0720: reopening flushed file at 1731613068256 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@505d5b54: reopening flushed file at 1731613068259 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3462deda: reopening flushed file at 1731613068263 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58c4c290: reopening flushed file at 1731613068267 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1319ms, sequenceid=90, compaction requested=false at 1731613068271 (+4 ms)Writing region close event to WAL at 1731613068272 (+1 ms)Running coprocessor post-close hooks at 1731613068275 (+3 ms)Closed at 1731613068276 (+1 ms) 2024-11-14T19:37:48,276 DEBUG [RS_CLOSE_META-regionserver/867b237d0fa7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T19:37:48,353 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(976): stopping server 867b237d0fa7,38601,1731612889637; all regions closed. 2024-11-14T19:37:48,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741834_1010 (size=25344) 2024-11-14T19:37:48,359 DEBUG [RS:0;867b237d0fa7:38601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/oldWALs 2024-11-14T19:37:48,359 INFO [RS:0;867b237d0fa7:38601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 867b237d0fa7%2C38601%2C1731612889637.meta:.meta(num 1731612893588) 2024-11-14T19:37:48,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741833_1009 (size=18893926) 2024-11-14T19:37:48,363 DEBUG [RS:0;867b237d0fa7:38601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/oldWALs 2024-11-14T19:37:48,363 INFO [RS:0;867b237d0fa7:38601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 867b237d0fa7%2C38601%2C1731612889637:(num 1731612892845) 2024-11-14T19:37:48,363 DEBUG [RS:0;867b237d0fa7:38601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T19:37:48,363 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T19:37:48,363 INFO [RS:0;867b237d0fa7:38601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T19:37:48,363 INFO [RS:0;867b237d0fa7:38601 {}] hbase.ChoreService(370): Chore service for: regionserver/867b237d0fa7:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T19:37:48,363 INFO [RS:0;867b237d0fa7:38601 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T19:37:48,363 INFO [regionserver/867b237d0fa7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T19:37:48,364 INFO [RS:0;867b237d0fa7:38601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38601 2024-11-14T19:37:48,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T19:37:48,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/867b237d0fa7,38601,1731612889637 2024-11-14T19:37:48,377 INFO [RS:0;867b237d0fa7:38601 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T19:37:48,385 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [867b237d0fa7,38601,1731612889637] 2024-11-14T19:37:48,393 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/867b237d0fa7,38601,1731612889637 already deleted, retry=false 2024-11-14T19:37:48,394 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 867b237d0fa7,38601,1731612889637 expired; onlineServers=0 2024-11-14T19:37:48,394 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '867b237d0fa7,38019,1731612888274' ***** 2024-11-14T19:37:48,394 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T19:37:48,394 INFO [M:0;867b237d0fa7:38019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T19:37:48,394 INFO [M:0;867b237d0fa7:38019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T19:37:48,394 DEBUG [M:0;867b237d0fa7:38019 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T19:37:48,394 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T19:37:48,394 DEBUG [M:0;867b237d0fa7:38019 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T19:37:48,394 DEBUG [master/867b237d0fa7:0:becomeActiveMaster-HFileCleaner.small.0-1731612892243 {}] cleaner.HFileCleaner(306): Exit Thread[master/867b237d0fa7:0:becomeActiveMaster-HFileCleaner.small.0-1731612892243,5,FailOnTimeoutGroup] 2024-11-14T19:37:48,394 DEBUG [master/867b237d0fa7:0:becomeActiveMaster-HFileCleaner.large.0-1731612892239 {}] cleaner.HFileCleaner(306): Exit Thread[master/867b237d0fa7:0:becomeActiveMaster-HFileCleaner.large.0-1731612892239,5,FailOnTimeoutGroup] 2024-11-14T19:37:48,394 INFO [M:0;867b237d0fa7:38019 {}] hbase.ChoreService(370): Chore service for: master/867b237d0fa7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T19:37:48,395 INFO [M:0;867b237d0fa7:38019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T19:37:48,395 DEBUG [M:0;867b237d0fa7:38019 {}] master.HMaster(1795): Stopping service threads 2024-11-14T19:37:48,395 INFO [M:0;867b237d0fa7:38019 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T19:37:48,395 INFO [M:0;867b237d0fa7:38019 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T19:37:48,395 ERROR [M:0;867b237d0fa7:38019 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1420519831) connection to localhost/127.0.0.1:36419 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:36419,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-14T19:37:48,395 INFO [M:0;867b237d0fa7:38019 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T19:37:48,395 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T19:37:48,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T19:37:48,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T19:37:48,402 DEBUG [M:0;867b237d0fa7:38019 {}] zookeeper.ZKUtil(347): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T19:37:48,402 WARN [M:0;867b237d0fa7:38019 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T19:37:48,402 INFO [M:0;867b237d0fa7:38019 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/.lastflushedseqids 2024-11-14T19:37:48,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742451_1627 (size=72) 2024-11-14T19:37:48,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T19:37:48,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38601-0x1013c058da80001, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T19:37:48,486 INFO [RS:0;867b237d0fa7:38601 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T19:37:48,486 INFO [RS:0;867b237d0fa7:38601 {}] regionserver.HRegionServer(1031): Exiting; stopping=867b237d0fa7,38601,1731612889637; zookeeper connection closed. 2024-11-14T19:37:48,487 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@62addd19 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@62addd19 2024-11-14T19:37:48,487 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T19:37:48,812 INFO [M:0;867b237d0fa7:38019 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T19:37:48,812 INFO [M:0;867b237d0fa7:38019 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T19:37:48,812 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T19:37:48,812 INFO [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:37:48,812 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:37:48,812 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T19:37:48,812 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:37:48,812 INFO [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=781.94 KB heapSize=958.30 KB 2024-11-14T19:37:48,833 DEBUG [M:0;867b237d0fa7:38019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16b7f1c6ab0d49bd970be22d8591b648 is 82, key is hbase:meta,,1/info:regioninfo/1731612893803/Put/seqid=0 2024-11-14T19:37:48,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742452_1628 (size=5672) 2024-11-14T19:37:48,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T19:37:48,897 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T19:37:48,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-14T19:37:49,238 INFO [M:0;867b237d0fa7:38019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2181 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16b7f1c6ab0d49bd970be22d8591b648 2024-11-14T19:37:49,266 DEBUG [M:0;867b237d0fa7:38019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edbbe4fca2ab4e11a84e654adc75128b is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1731612920177/Put/seqid=0 2024-11-14T19:37:49,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742453_1629 (size=45073) 2024-11-14T19:37:49,672 INFO [M:0;867b237d0fa7:38019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=781.38 KB at sequenceid=2181 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edbbe4fca2ab4e11a84e654adc75128b 2024-11-14T19:37:49,675 INFO [M:0;867b237d0fa7:38019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for edbbe4fca2ab4e11a84e654adc75128b 2024-11-14T19:37:49,690 DEBUG [M:0;867b237d0fa7:38019 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f489ecca427f40028bdab02e5807bdd7 is 69, key is 867b237d0fa7,38601,1731612889637/rs:state/1731612892349/Put/seqid=0 2024-11-14T19:37:49,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073742454_1630 (size=5156) 2024-11-14T19:37:50,098 INFO [M:0;867b237d0fa7:38019 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2181 (bloomFilter=true), to=hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f489ecca427f40028bdab02e5807bdd7 2024-11-14T19:37:50,103 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/16b7f1c6ab0d49bd970be22d8591b648 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16b7f1c6ab0d49bd970be22d8591b648 2024-11-14T19:37:50,107 INFO [M:0;867b237d0fa7:38019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/16b7f1c6ab0d49bd970be22d8591b648, entries=8, sequenceid=2181, filesize=5.5 K 2024-11-14T19:37:50,108 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edbbe4fca2ab4e11a84e654adc75128b as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/edbbe4fca2ab4e11a84e654adc75128b 2024-11-14T19:37:50,111 INFO [M:0;867b237d0fa7:38019 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for edbbe4fca2ab4e11a84e654adc75128b 2024-11-14T19:37:50,111 INFO [M:0;867b237d0fa7:38019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/edbbe4fca2ab4e11a84e654adc75128b, entries=168, sequenceid=2181, filesize=44.0 K 2024-11-14T19:37:50,112 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f489ecca427f40028bdab02e5807bdd7 as hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f489ecca427f40028bdab02e5807bdd7 2024-11-14T19:37:50,115 INFO [M:0;867b237d0fa7:38019 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36419/user/jenkins/test-data/b6238124-f2d7-da2a-9dab-50c2bea61041/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f489ecca427f40028bdab02e5807bdd7, entries=1, sequenceid=2181, filesize=5.0 K 2024-11-14T19:37:50,115 INFO [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(3140): Finished flush of dataSize ~781.94 KB/800704, heapSize ~958 KB/980992, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2181, compaction requested=false 2024-11-14T19:37:50,117 INFO [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T19:37:50,117 DEBUG [M:0;867b237d0fa7:38019 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731613068812Disabling compacts and flushes for region at 1731613068812Disabling writes for close at 1731613068812Obtaining lock to block concurrent updates at 1731613068812Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731613068812Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=800704, getHeapSize=981232, getOffHeapSize=0, getCellsCount=2374 at 1731613068812Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731613068813 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731613068813Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731613068833 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731613068833Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731613069244 (+411 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731613069265 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731613069265Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731613069675 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731613069689 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731613069689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1866f91: reopening flushed file at 1731613070102 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c8e6af6: reopening flushed file at 1731613070107 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33c508fa: reopening flushed file at 1731613070111 (+4 ms)Finished flush of dataSize ~781.94 KB/800704, heapSize ~958 KB/980992, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1303ms, sequenceid=2181, compaction requested=false at 1731613070115 (+4 ms)Writing region close event to WAL at 1731613070117 (+2 ms)Closed at 1731613070117 2024-11-14T19:37:50,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43995 is added to blk_1073741830_1006 (size=943660) 2024-11-14T19:37:50,121 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T19:37:50,121 INFO [M:0;867b237d0fa7:38019 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T19:37:50,121 INFO [M:0;867b237d0fa7:38019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38019 2024-11-14T19:37:50,121 INFO [M:0;867b237d0fa7:38019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T19:37:50,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T19:37:50,256 INFO [M:0;867b237d0fa7:38019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T19:37:50,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38019-0x1013c058da80000, quorum=127.0.0.1:51734, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T19:37:50,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@42a79998{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T19:37:50,267 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4975be1c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T19:37:50,267 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T19:37:50,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74fde7a3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T19:37:50,267 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de8243e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/hadoop.log.dir/,STOPPED} 2024-11-14T19:37:50,270 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T19:37:50,270 WARN [BP-1729087364-172.17.0.2-1731612883524 heartbeating to localhost/127.0.0.1:36419 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T19:37:50,270 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T19:37:50,270 WARN [BP-1729087364-172.17.0.2-1731612883524 heartbeating to localhost/127.0.0.1:36419 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1729087364-172.17.0.2-1731612883524 (Datanode Uuid 61f6364d-ad36-44dd-b7fe-2d5da06a630e) service to localhost/127.0.0.1:36419 2024-11-14T19:37:50,273 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389/data/data1/current/BP-1729087364-172.17.0.2-1731612883524 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T19:37:50,273 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/cluster_ec52a890-a6d9-e807-8688-03b9ebcaf389/data/data2/current/BP-1729087364-172.17.0.2-1731612883524 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T19:37:50,274 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T19:37:50,281 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T19:37:50,282 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T19:37:50,282 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T19:37:50,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T19:37:50,282 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5ab35520-12e4-8b90-88c7-c25876d96657/hadoop.log.dir/,STOPPED} 2024-11-14T19:37:50,295 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T19:37:50,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down